diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a8cc420f..1b6c42d7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -303,6 +303,9 @@ jobs: working-directory: cli-connection-reader run: node --test + - name: Verify portable private compiler authority + run: node --test cli/build-windows-internal-repro.test.mjs cli/create-windows-internal-repro-inputs.test.mjs cli/windows-compiler-closure.test.mjs + # The repo's Python tests. Same hole as `cli-connection-reader` above, as # `steel-detailer-lookup`, and as the six .NET suites below — a real regression # suite that NO workflow ran. `20-agents/aeco/visualization/blender/tests/` @@ -365,7 +368,8 @@ jobs: bridge-windows-packaged: name: connection-reader packaged RVT/IFC harness runs-on: windows-latest - timeout-minutes: 15 + # Two fresh compiler/SDK copies plus debug-event audits precede the packaged host build. + timeout-minutes: 45 steps: - uses: actions/checkout@v6 @@ -375,9 +379,20 @@ jobs: cache: npm cache-dependency-path: cli-connection-reader/package-lock.json - - uses: dtolnay/rust-toolchain@stable + - uses: dtolnay/rust-toolchain@master + with: + toolchain: 1.95.0 + + - name: Configure the native x64 compiler environment + uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1 with: - toolchain: 1.88.0 + arch: x64 + + - name: Verify private builder authority + run: node --test cli/build-windows-internal-repro.test.mjs cli/create-windows-internal-repro-inputs.test.mjs cli/windows-compiler-closure.test.mjs + + - name: Verify native vendor-path reproducibility + run: node cli/windows-vendor-repro.native.mjs - name: Build source AWARE host run: cargo build --manifest-path cli/Cargo.toml --locked diff --git a/20-agents/aeco/engineering/model-reference-reader/manifest.yaml b/20-agents/aeco/engineering/model-reference-reader/manifest.yaml index 28e6488b5..a78934724 100644 --- a/20-agents/aeco/engineering/model-reference-reader/manifest.yaml +++ b/20-agents/aeco/engineering/model-reference-reader/manifest.yaml @@ -1,5 +1,5 @@ agent: model-reference-reader -version: 0.4.0 +version: 0.5.0 display-name: Authenticated RVT Reference Reader description: | Convert a local Revit project into deterministic, authenticated reference-model artifacts without @@ -52,6 +52,10 @@ commands: type: string required: false description: Installer-enrolled absolute authority-store path passed only to a managed-cloud v2 provider. + reader-schema-version: + type: string + required: false + description: Exact AWARE reader contract; defaults to model-reference-reader/v1, or opt into model-reference-reader/v2. outputs: type: single schema: @@ -91,6 +95,14 @@ commands: type: string required: false description: Installer-enrolled absolute authority-store path passed only to a managed-cloud v2 provider. + reader-schema-version: + type: string + required: false + description: Exact AWARE reader contract; defaults to model-reference-reader/v1, or opt into model-reference-reader/v2. + property-expansion-limits: + type: object + required: false + description: Optional lower ceilings for expanded property rows and canonical property bytes under reader v2. expected-signer-sha256: type: string required: true @@ -136,6 +148,14 @@ commands: type: string required: false description: Installer-enrolled absolute authority-store path passed only to a managed-cloud v2 provider. + reader-schema-version: + type: string + required: false + description: Exact AWARE reader contract; defaults to model-reference-reader/v1, or opt into model-reference-reader/v2. + property-expansion-limits: + type: object + required: false + description: Optional lower ceilings for expanded property rows and canonical property bytes under reader v2. expected-signer-sha256: type: string required: true @@ -182,6 +202,14 @@ commands: type: string required: false description: Installer-enrolled absolute authority-store path passed only to a managed-cloud v2 provider. + reader-schema-version: + type: string + required: false + description: Exact AWARE reader contract; defaults to model-reference-reader/v1, or opt into model-reference-reader/v2. + property-expansion-limits: + type: object + required: false + description: Optional lower ceilings for expanded property rows and canonical property bytes under reader v2. expected-signer-sha256: type: string required: true diff --git a/cli-connection-reader/build-internal-repro.mjs b/cli-connection-reader/build-internal-repro.mjs new file mode 100644 index 000000000..f61ef0872 --- /dev/null +++ b/cli-connection-reader/build-internal-repro.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +// Fail-closed entrypoint for the private Windows reproducibility proof. Real machine paths live only +// in the per-builder locator; the canonical manifest and receipt contain logical IDs and digests. +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync, realpathSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { buildConnectionReader, canonicalJson, READER_BUILD_SETTINGS, sha256File } from './build.mjs'; + +const here = dirname(fileURLToPath(import.meta.url)); +const SHA256 = /^[0-9a-f]{64}$/; +const POISONED_EXACT = new Set([ + 'RUSTFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'CC', 'CFLAGS', 'CL', 'LINK', 'LIB', 'INCLUDE', + 'NODE_OPTIONS', 'ESBUILD_BINARY_PATH', 'GOOGLE_CLIENT_SECRET', 'AWARE_GOOGLE_CLIENT_SECRET', +]); +const POISONED_PREFIXES = ['npm_config_', 'DOTNET_', 'COREHOST_']; + +const digestText = (text) => createHash('sha256').update(text).digest('hex'); +const readJson = (path) => JSON.parse(readFileSync(path, 'utf8')); + +export function rejectedAmbientKeys(env) { + // Windows environment names are case-insensitive, so an ambient + // `esbuild_binary_path` still resolves as ESBUILD_BINARY_PATH. Match the outer + // builder: fold case before the exact lookup, and reject names that collide + // only by case, since which one a child reads is not ours to decide. + const counts = Object.keys(env).reduce( + (map, key) => map.set(key.toLowerCase(), (map.get(key.toLowerCase()) ?? 0) + 1), new Map()); + return Object.keys(env).filter((key) => counts.get(key.toLowerCase()) > 1 + || POISONED_EXACT.has(key.toUpperCase()) + || POISONED_PREFIXES.some((prefix) => key.toLowerCase().startsWith(prefix.toLowerCase()))) + .sort(); +} + +function verifyRecord(id, record, locator) { + if (!record || record.id !== id || !SHA256.test(record.sha256 ?? '')) throw new Error(`invalid tool record ${id}`); + const path = locator?.tools?.[id]; + if (typeof path !== 'string' || !existsSync(path)) throw new Error(`missing local path for tool ${id}`); + const actual = sha256File(path); + if (actual !== record.sha256) throw new Error(`tool digest mismatch for ${id}: ${actual}`); + return resolve(path); +} + +export function verifyInternalInputs({ manifest, locator, env = process.env }) { + if (manifest?.schema !== 'aware-windows-repro-builder/v1') throw new Error('unsupported builder manifest schema'); + if (locator?.schema !== 'aware-windows-repro-locator/v1') throw new Error('unsupported builder locator schema'); + const poison = rejectedAmbientKeys(env); + if (poison.length) throw new Error(`ambient build authority is forbidden: ${poison.join(', ')}`); + if (manifest.platform !== 'win32' || manifest.arch !== 'x64' || manifest.nodeVersion !== '24.14.0') { + throw new Error('builder manifest must pin Windows x64 and Node 24.14.0'); + } + if (canonicalJson(manifest.settings) !== canonicalJson(READER_BUILD_SETTINGS)) { + throw new Error('reader build settings differ from the closed implementation'); + } + const paths = Object.fromEntries(['node', 'postject', 'web-ifc-wasm'].map((id) => [id, + verifyRecord(id, manifest.tools?.[id], locator), + ])); + if (process.platform !== 'win32' || process.arch !== 'x64' || process.versions.node !== manifest.nodeVersion) { + throw new Error(`running Node must be exactly ${manifest.nodeVersion} on Windows x64`); + } + if (realpathSync(paths.node) !== realpathSync(process.execPath)) throw new Error('the verified Node is not the running Node'); + for (const [name, path] of Object.entries({ + 'reader-package-lock': join(here, 'package-lock.json'), + 'aware-cargo-lock': join(here, '..', 'cli', 'Cargo.lock'), + })) { + const expected = manifest.inputs?.[name]; + if (!SHA256.test(expected ?? '') || sha256File(path) !== expected) throw new Error(`input digest mismatch for ${name}`); + } + if (!SHA256.test(manifest.source?.bundleSha256 ?? '') || !/^[0-9a-f]{40}$/.test(manifest.source?.commit ?? '') + || !/^[0-9a-f]{40}$/.test(manifest.source?.tree ?? '')) throw new Error('invalid source identity in builder manifest'); + return paths; +} + +function parseArgs(argv) { + const out = {}; + for (let i = 0; i < argv.length; i += 2) { + if (!argv[i]?.startsWith('--') || argv[i + 1] == null) throw new Error('arguments are --manifest FILE --locator FILE --output DIR'); + out[argv[i].slice(2)] = argv[i + 1]; + } + if (!out.manifest || !out.locator || !out.output) throw new Error('arguments are --manifest FILE --locator FILE --output DIR'); + return out; +} + +export async function runInternalReaderBuild({ manifestPath, locatorPath, outputDir, env = process.env }) { + const manifestText = canonicalJson(readJson(manifestPath)); + const manifest = JSON.parse(manifestText); + const locator = readJson(locatorPath); + const paths = verifyInternalInputs({ manifest, locator, env }); + const receipt = { + schema: 'aware-connection-reader-build-receipt/v1', + buildId: digestText(manifestText), + builderManifestSha256: digestText(manifestText), + source: manifest.source, + inputs: manifest.inputs, + settings: manifest.settings, + tools: Object.fromEntries(Object.entries(manifest.tools).map(([id, record]) => [id, { + id: record.id, sha256: record.sha256, + }])), + commands: { + bundle: ' model-dispatcher.mjs -> /bundle.cjs', + sea: ' --experimental-sea-config sea-config.json [cwd=]', + inject: ' aware-connection-reader.exe NODE_SEA_BLOB sea-prep.blob [cwd=]', + }, + rootTokens: ['', '', '', '', ''], + }; + return buildConnectionReader({ + outputDir, nodePath: paths.node, postjectPath: paths.postject, + wasmPath: paths['web-ifc-wasm'], receipt, verifiedExternalTools: true, + }); +} + +if (realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1])) { + const args = parseArgs(process.argv.slice(2)); + await runInternalReaderBuild({ manifestPath: args.manifest, locatorPath: args.locator, outputDir: args.output }); +} diff --git a/cli-connection-reader/build-internal-repro.test.mjs b/cli-connection-reader/build-internal-repro.test.mjs new file mode 100644 index 000000000..13b2e5e33 --- /dev/null +++ b/cli-connection-reader/build-internal-repro.test.mjs @@ -0,0 +1,85 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { buildConnectionReader, canonicalJson, READER_BUILD_SETTINGS } from './build.mjs'; +import { rejectedAmbientKeys, verifyInternalInputs } from './build-internal-repro.mjs'; + +const sha = (bytes) => createHash('sha256').update(bytes).digest('hex'); + +test('SEA configuration is path-independent and every injection operand is relative to output cwd', () => { + const text = readFileSync(new URL('./build.mjs', import.meta.url), 'utf8'); + assert.match(text, /main: 'bundle\.cjs'/); + assert.match(text, /output: 'sea-prep\.blob'/); + assert.match(text, /cwd: outputDir/); + assert.match(text, /EXE_NAME, READER_BUILD_SETTINGS\.sea\.section, 'sea-prep\.blob'/); + assert.match(text, /'--sentinel-fuse', READER_BUILD_SETTINGS\.sea\.sentinelFuse/); + assert.doesNotMatch(text, /main:\s*join\(outputDir/); + assert.doesNotMatch(text, /output:\s*join\(outputDir/); +}); + +test('canonical receipt JSON is independent of insertion order', () => { + assert.equal(canonicalJson({ z: 1, a: { y: 2, x: 3 } }), canonicalJson({ a: { x: 3, y: 2 }, z: 1 })); +}); + +test('build cleanup refuses to own the reader source root', async () => { + const readerRoot = new URL('.', import.meta.url).pathname.replace(/^\/(?:[A-Za-z]:)/, (value) => value.slice(1)); + await assert.rejects(() => buildConnectionReader({ outputDir: readerRoot }), /must not be the reader source root/); +}); + +test('ambient compiler, Node, npm, dotnet, and credential authority is rejected', () => { + assert.deepEqual(rejectedAmbientKeys({ Path: 'ok', RUSTFLAGS: 'poison', npm_config_cache: 'x', DOTNET_ROOT: 'x' }), + ['DOTNET_ROOT', 'RUSTFLAGS', 'npm_config_cache']); + assert.deepEqual(rejectedAmbientKeys({ NODE_OPTIONS: '--require evil', AWARE_GOOGLE_CLIENT_SECRET: 'secret' }), + ['AWARE_GOOGLE_CLIENT_SECRET', 'NODE_OPTIONS']); +}); + +test('ambient authority is rejected by folded case and by case-only collisions', () => { + // Windows resolves process.env.ESBUILD_BINARY_PATH from an ambient + // `esbuild_binary_path`, so a case-sensitive exact match let esbuild run an + // undeclared binary inside a supposedly closed build. + assert.deepEqual(rejectedAmbientKeys({ esbuild_binary_path: 'C:\evil.exe' }), ['esbuild_binary_path']); + assert.deepEqual(rejectedAmbientKeys({ NoDe_OpTiOnS: '--require evil' }), ['NoDe_OpTiOnS']); + // Two spellings of one name: which one a child reads is not ours to decide. + assert.deepEqual(rejectedAmbientKeys({ Path: 'a', PATH: 'b' }), ['PATH', 'Path']); + // A single ordinary Path is still authority the reader needs. + assert.deepEqual(rejectedAmbientKeys({ Path: 'ok' }), []); +}); + +test('tool verification goes red when a declared tool byte changes', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-repro-test-')); + const tool = join(root, 'tool.bin'); writeFileSync(tool, 'one'); + const manifest = { + schema: 'aware-windows-repro-builder/v1', platform: 'win32', arch: 'x64', nodeVersion: '24.14.0', + settings: READER_BUILD_SETTINGS, + source: { bundleSha256: 'a'.repeat(64), commit: 'b'.repeat(40), tree: 'c'.repeat(40) }, + tools: { + node: { id: 'node', sha256: sha('one') }, + postject: { id: 'postject', sha256: sha('one') }, + 'web-ifc-wasm': { id: 'web-ifc-wasm', sha256: sha('one') }, + }, + inputs: { 'reader-package-lock': 'd'.repeat(64), 'aware-cargo-lock': 'e'.repeat(64) }, + }; + const locator = { schema: 'aware-windows-repro-locator/v1', tools: { node: tool, postject: tool, 'web-ifc-wasm': tool } }; + writeFileSync(tool, 'two'); + assert.throws(() => verifyInternalInputs({ manifest, locator, env: {} }), /tool digest mismatch/); +}); + +test('controlled reader rejects omitted or changed byte-affecting settings', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-repro-settings-')); + const tool = join(root, 'tool.bin'); writeFileSync(tool, 'one'); + const manifest = { + schema: 'aware-windows-repro-builder/v1', platform: 'win32', arch: 'x64', nodeVersion: '24.14.0', + source: { bundleSha256: 'a'.repeat(64), commit: 'b'.repeat(40), tree: 'c'.repeat(40) }, + tools: Object.fromEntries(['node', 'postject', 'web-ifc-wasm'].map((id) => [id, { id, sha256: sha('one') }])), + inputs: { 'reader-package-lock': 'd'.repeat(64), 'aware-cargo-lock': 'e'.repeat(64) }, + }; + const locator = { schema: 'aware-windows-repro-locator/v1', tools: { node: tool, postject: tool, 'web-ifc-wasm': tool } }; + assert.throws(() => verifyInternalInputs({ manifest, locator, env: {} }), /reader build settings differ/); + assert.throws(() => verifyInternalInputs({ + manifest: { ...manifest, settings: { ...READER_BUILD_SETTINGS, sea: { ...READER_BUILD_SETTINGS.sea, section: 'evil' } } }, + locator, env: {}, + }), /reader build settings differ/); +}); diff --git a/cli-connection-reader/build.mjs b/cli-connection-reader/build.mjs index 105de89dd..59ddee5bd 100644 --- a/cli-connection-reader/build.mjs +++ b/cli-connection-reader/build.mjs @@ -1,69 +1,110 @@ -// build.mjs — package aware-connection-reader as a standalone Windows exe via Node's built-in SEA. +// build.mjs — package aware-connection-reader as a path-independent Windows SEA. // -// Why SEA + esbuild: the AWARE runtime spawns `~/.aware/bridges/aware-connection-reader.exe` by -// absolute path (see cli/src/commands/sidecar.rs), so the bridge must ship as ONE real .exe with no -// node_modules beside it. SEA embeds a single bundled script into a copy of node.exe; esbuild first -// inlines index.mjs + web-ifc's JS into that single file. web-ifc's .wasm is read from disk at run -// time (index.mjs SetWasmPath → the exe's dir), so it ships as a sibling file, not inlined. +// The ordinary developer entrypoint deliberately stays usable by the public release workflow. The +// stricter internal reproducibility boundary lives in build-internal-repro.mjs and supplies the +// verified tool/input identities plus the canonical receipt written by this module. import { build } from 'esbuild'; import { execFileSync } from 'node:child_process'; -import { copyFileSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; +import { createHash } from 'node:crypto'; +import { + copyFileSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync, +} from 'node:fs'; +import { basename, dirname, isAbsolute, join, relative, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { READER_BUILD_SETTINGS } from './repro-settings.mjs'; + +export { READER_BUILD_SETTINGS } from './repro-settings.mjs'; const here = dirname(fileURLToPath(import.meta.url)); -const dist = join(here, 'dist'); -const exeName = 'aware-connection-reader.exe'; -const FUSE = 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2'; // Node's standard SEA sentinel fuse +const EXE_NAME = 'aware-connection-reader.exe'; + +export const sha256File = (path) => createHash('sha256').update(readFileSync(path)).digest('hex'); + +function canonical(value) { + if (Array.isArray(value)) return value.map(canonical); + if (!value || typeof value !== 'object') return value; + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])); +} + +export function canonicalJson(value) { + return `${JSON.stringify(canonical(value), null, 2)}\n`; +} + +function under(root, path) { + const rel = relative(resolve(root), resolve(path)); + return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel)); +} + +export async function buildConnectionReader(options = {}) { + const outputDir = resolve(options.outputDir ?? join(here, 'dist')); + const nodePath = resolve(options.nodePath ?? process.execPath); + const postjectPath = resolve(options.postjectPath ?? join(here, 'node_modules', 'postject', 'dist', 'cli.js')); + const wasmPath = resolve(options.wasmPath ?? join(here, 'node_modules', 'web-ifc', 'web-ifc-node.wasm')); + const receipt = options.receipt ?? null; + + const outputOwnsSource = relative(outputDir, here); + if (outputOwnsSource === '' || (!outputOwnsSource.startsWith('..') && !isAbsolute(outputOwnsSource))) { + throw new Error('SEA output root must not be the reader source root or one of its ancestors'); + } + if (process.platform !== 'win32' || process.arch !== 'x64') { + throw new Error('aware-connection-reader SEA build requires Windows x64'); + } + if (basename(nodePath).toLowerCase() !== 'node.exe') throw new Error('SEA base runtime must be node.exe'); + for (const path of [postjectPath, wasmPath]) { + if (!options.verifiedExternalTools && !under(here, path)) { + throw new Error(`reader dependency escaped the package root without verified external-tool authority: ${path}`); + } + } + + rmSync(outputDir, { recursive: true, force: true }); + mkdirSync(outputDir, { recursive: true }); -rmSync(dist, { recursive: true, force: true }); -mkdirSync(dist, { recursive: true }); + console.log('[build] bundling with esbuild…'); + await build({ + absWorkingDir: here, + entryPoints: ['model-dispatcher.mjs'], + bundle: true, + platform: READER_BUILD_SETTINGS.bundle.platform, + format: READER_BUILD_SETTINGS.bundle.format, + target: READER_BUILD_SETTINGS.bundle.target, + outfile: join(outputDir, 'bundle.cjs'), + }); -// 1. Bundle ESM entry + web-ifc into one CJS file (SEA embeds a single file; node_modules aren't shipped). -console.log('[build] bundling with esbuild…'); -await build({ - entryPoints: [join(here, 'model-dispatcher.mjs')], - bundle: true, - platform: 'node', - format: 'cjs', - target: 'node20', - outfile: join(dist, 'bundle.cjs'), -}); + // Relative names plus outputDir as cwd are intentional. Absolute roots in this file are embedded + // into the SEA blob by Node even though the bundled JavaScript itself is byte-identical. + console.log('[build] generating SEA blob…'); + writeFileSync(join(outputDir, 'sea-config.json'), canonicalJson({ + disableExperimentalSEAWarning: READER_BUILD_SETTINGS.sea.disableExperimentalWarning, + main: 'bundle.cjs', + output: 'sea-prep.blob', + }), 'utf8'); + execFileSync(nodePath, ['--experimental-sea-config', 'sea-config.json'], { + cwd: outputDir, stdio: 'inherit', windowsHide: true, + }); -// 2. Generate the SEA blob from the bundle. -console.log('[build] generating SEA blob…'); -const seaConfig = join(dist, 'sea-config.json'); -writeFileSync( - seaConfig, - JSON.stringify({ - main: join(dist, 'bundle.cjs'), - output: join(dist, 'sea-prep.blob'), - disableExperimentalSEAWarning: true, - }), -); -execFileSync(process.execPath, ['--experimental-sea-config', seaConfig], { stdio: 'inherit' }); + console.log('[build] injecting blob into exe…'); + const exe = join(outputDir, EXE_NAME); + copyFileSync(nodePath, exe); + execFileSync(nodePath, [ + postjectPath, EXE_NAME, READER_BUILD_SETTINGS.sea.section, 'sea-prep.blob', + '--sentinel-fuse', READER_BUILD_SETTINGS.sea.sentinelFuse, + ], { cwd: outputDir, stdio: 'inherit', windowsHide: true }); + copyFileSync(wasmPath, join(outputDir, 'web-ifc-node.wasm')); -// 3. Copy node.exe and inject the blob with postject. -console.log('[build] injecting blob into exe…'); -const exe = join(dist, exeName); -copyFileSync(process.execPath, exe); -execFileSync( - process.execPath, - [ - join(here, 'node_modules', 'postject', 'dist', 'cli.js'), - exe, - 'NODE_SEA_BLOB', - join(dist, 'sea-prep.blob'), - '--sentinel-fuse', - FUSE, - ], - { stdio: 'inherit' }, -); + const outputs = Object.fromEntries([ + 'bundle.cjs', 'sea-prep.blob', EXE_NAME, 'web-ifc-node.wasm', + ].map((name) => [name, { sha256: sha256File(join(outputDir, name)), size: readFileSync(join(outputDir, name)).length }])); + if (receipt) writeFileSync(join(outputDir, 'build-receipt.json'), canonicalJson({ ...receipt, outputs }), 'utf8'); + console.log(`[build] done → ${exe} (+ web-ifc-node.wasm)`); + return { outputDir, outputs, receiptPath: receipt ? join(outputDir, 'build-receipt.json') : null }; +} -// 4. Ship web-ifc's .wasm alongside the exe (read at run time via SetWasmPath). -copyFileSync( - join(here, 'node_modules', 'web-ifc', 'web-ifc-node.wasm'), - join(dist, 'web-ifc-node.wasm'), -); +function isEntryModule() { + try { return realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1]); } + catch { return fileURLToPath(import.meta.url) === resolve(process.argv[1] || ''); } +} -console.log(`[build] done → ${join(dist, exeName)} (+ web-ifc-node.wasm)`); +if (isEntryModule()) { + if (process.argv.length !== 2) throw new Error('build.mjs accepts no arguments; use build-internal-repro.mjs for the controlled build'); + await buildConnectionReader(); +} diff --git a/cli-connection-reader/model-contract.mjs b/cli-connection-reader/model-contract.mjs index ba653434d..8e53e1870 100644 --- a/cli-connection-reader/model-contract.mjs +++ b/cli-connection-reader/model-contract.mjs @@ -1,7 +1,10 @@ import { createHash, randomUUID } from 'node:crypto'; import { isUtf8 } from 'node:buffer'; -export const READER_SCHEMA_VERSION = 'model-reference-reader/v1'; +export const READER_SCHEMA_VERSION_V1 = 'model-reference-reader/v1'; +export const READER_SCHEMA_VERSION_V2 = 'model-reference-reader/v2'; +// Kept as v1 for callers that have not opted into the additive v2 contract. +export const READER_SCHEMA_VERSION = READER_SCHEMA_VERSION_V1; const SHA256 = /^[0-9a-f]{64}$/; const PLAIN = Object.getPrototypeOf({}); @@ -33,6 +36,11 @@ export const MODEL_LIMITS = Object.freeze({ maxCommandResponseBytes: { default: 1024 * 1024, hard: 1024 * 1024 }, }); +export const PROPERTY_EXPANSION_LIMITS = Object.freeze({ + maxExpandedPropertyRows: { default: 100_000, hard: 2_000_000 }, + maxCanonicalPropertyBytes: { default: 16 * 1024 * 1024, hard: 32 * 1024 * 1024 }, +}); + export class ModelReaderError extends Error { constructor(code, phase, retryable, message, unsafeDetails = undefined) { super(message); @@ -241,14 +249,31 @@ export function lowerableLimits(overrides = {}) { return result; } +export function lowerablePropertyExpansionLimits(overrides = {}) { + assertClosedObject(overrides, [], Object.keys(PROPERTY_EXPANSION_LIMITS), 'propertyExpansionLimits'); + const result = {}; + for (const [name, range] of Object.entries(PROPERTY_EXPANSION_LIMITS)) { + const selected = Object.hasOwn(overrides, name) ? overrides[name] : range.default; + if (!Number.isSafeInteger(selected) || selected <= 0 || selected > range.hard) { + throw new TypeError(`${name} exceeds its hard ceiling`); + } + result[name] = selected; + } + return result; +} + export function buildCanonicalRequest(options = {}) { const limits = lowerableLimits(options.limits); const protocolVersion = options.protocolVersion ?? '1'; if (!['1', '2'].includes(protocolVersion)) throw new TypeError('protocolVersion must be 1 or 2'); - return { + const readerSchemaVersion = options.readerSchemaVersion ?? READER_SCHEMA_VERSION_V1; + if (![READER_SCHEMA_VERSION_V1, READER_SCHEMA_VERSION_V2].includes(readerSchemaVersion)) { + throw new TypeError('readerSchemaVersion is unsupported'); + } + const request = { schemaVersion: '1', protocolVersion, - readerSchemaVersion: READER_SCHEMA_VERSION, + readerSchemaVersion, format: 'rvt', documentKind: 'revit-project', activeScenePolicy: 'declared-active-scene-only', @@ -269,6 +294,17 @@ export function buildCanonicalRequest(options = {}) { conversionSettings: options.conversionSettings ?? {}, limits, }; + if (readerSchemaVersion === READER_SCHEMA_VERSION_V1) return request; + return { + ...request, + schemaVersion: '2', + metadata: { + ...request.metadata, + propertyValues: ['source-storage', 'provider-display'], + providerDisplayIdentity: 'excluded', + }, + propertyExpansionLimits: lowerablePropertyExpansionLimits(options.propertyExpansionLimits), + }; } export function requestSha256(request) { diff --git a/cli-connection-reader/model-contract.test.mjs b/cli-connection-reader/model-contract.test.mjs index 081a7b7b0..ad4c1b87d 100644 --- a/cli-connection-reader/model-contract.test.mjs +++ b/cli-connection-reader/model-contract.test.mjs @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { MODEL_LIMITS, + PROPERTY_EXPANSION_LIMITS, ModelReaderError, buildCanonicalRequest, buildProviderFingerprint, @@ -75,6 +76,27 @@ test('provider fingerprint is the exact seven-field JCS tuple', () => { } }); +test('reader v2 binds closed effective expansion limits while v1 canonical bytes remain unchanged', () => { + const v1 = buildCanonicalRequest(); + assert.equal(v1.schemaVersion, '1'); + assert.equal('propertyExpansionLimits' in v1, false); + const v2 = buildCanonicalRequest({ + readerSchemaVersion: 'model-reference-reader/v2', + propertyExpansionLimits: { maxExpandedPropertyRows: 12, maxCanonicalPropertyBytes: 4096 }, + }); + assert.equal(v2.schemaVersion, '2'); + assert.deepEqual(v2.propertyExpansionLimits, { maxExpandedPropertyRows: 12, maxCanonicalPropertyBytes: 4096 }); + assert.deepEqual(v2.metadata.propertyValues, ['source-storage', 'provider-display']); + assert.equal(v2.metadata.providerDisplayIdentity, 'excluded'); + assert.equal(PROPERTY_EXPANSION_LIMITS.maxExpandedPropertyRows.hard, 2_000_000); + assert.equal(PROPERTY_EXPANSION_LIMITS.maxCanonicalPropertyBytes.hard, 32 * 1024 * 1024); + assert.throws(() => buildCanonicalRequest({ + readerSchemaVersion: 'model-reference-reader/v2', + propertyExpansionLimits: { maxExpandedPropertyRows: 2_000_001 }, + }), /hard ceiling/); + assert.throws(() => buildCanonicalRequest({ readerSchemaVersion: 'model-reference-reader/v3' }), /unsupported/); +}); + test('managed-cloud provider fingerprint binds execution and exact destination', () => { const fingerprint = buildProviderFingerprint({ protocolVersion: '2', provider: 'fixture', engine: 'fixture-engine', engineVersion: '1.2.3', diff --git a/cli-connection-reader/model-metadata-v2.schema.json b/cli-connection-reader/model-metadata-v2.schema.json new file mode 100644 index 000000000..63ed0ef94 --- /dev/null +++ b/cli-connection-reader/model-metadata-v2.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://aware-aeco.org/schemas/model-metadata-v2.schema.json", + "title": "AWARE explicit Revit metadata v2", + "type": "object", + "additionalProperties": false, + "required": ["schemaVersion", "document", "types", "levels", "parameterGroups", "parameters", "elements", "relations"], + "properties": { + "schemaVersion": { "const": "2" }, + "document": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "id"], + "properties": { + "kind": { "const": "revit-project" }, + "id": { "type": "string", "minLength": 1 } + } + }, + "types": { "type": "array" }, + "levels": { "type": "array" }, + "parameterGroups": { "type": "array" }, + "parameters": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "unit", "valueEncoding", "valueType", "value"], + "properties": { + "id": { "type": "string", "pattern": "^[1-9][0-9]*$" }, + "name": { "type": "string" }, + "unit": { "type": ["string", "null"] }, + "valueEncoding": { "const": "provider-display" }, + "valueType": { "enum": ["string", "number"] }, + "value": { "type": ["string", "number"] } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "unit", "valueEncoding", "readable", "storageType", "value"], + "properties": { + "id": { "type": "string", "pattern": "^[1-9][0-9]*$" }, + "name": { "type": "string" }, + "unit": { "type": ["string", "null"] }, + "valueEncoding": { "const": "source-storage" }, + "readable": { "type": "boolean" }, + "storageType": { "enum": ["none", "boolean", "integer", "double", "string", "element-id"] }, + "value": {} + } + } + ] + } + }, + "elements": { "type": "array" }, + "relations": { "type": "array" } + } +} diff --git a/cli-connection-reader/model-provider.mjs b/cli-connection-reader/model-provider.mjs index 352461309..31cd44992 100644 --- a/cli-connection-reader/model-provider.mjs +++ b/cli-connection-reader/model-provider.mjs @@ -240,6 +240,7 @@ export async function describeProvider(options) { protocolVersion: describe.protocolVersion, provider: describe.provider, engine: describe.engine, engineVersion: describe.engineVersion, adapterBuildId: describe.adapterBuildId, adapterExecutableSha256: initialExecutable.sha256, + ...(options.readerSchemaVersion ? { readerSchemaVersion: options.readerSchemaVersion } : {}), ...(describe.protocolVersion === '2' ? { execution: describe.execution, destination: describe.destination } : {}), }); if (options.expectedProviderSha256 !== undefined) { @@ -272,6 +273,7 @@ export async function describeAndConvert(options) { protocolVersion: describe.protocolVersion, provider: describe.provider, engine: describe.engine, engineVersion: describe.engineVersion, adapterBuildId: describe.adapterBuildId, adapterExecutableSha256: initialExecutable.sha256, + ...(options.readerSchemaVersion ? { readerSchemaVersion: options.readerSchemaVersion } : {}), ...(describe.protocolVersion === '2' ? { execution: describe.execution, destination: describe.destination } : {}), }); if (options.expectedProviderSha256 !== undefined) { @@ -282,6 +284,8 @@ export async function describeAndConvert(options) { limits, protocolVersion: expectedProtocolVersion, conversionSettings: options.conversionSettings ?? {}, + readerSchemaVersion: options.readerSchemaVersion, + propertyExpansionLimits: options.propertyExpansionLimits, }); const outputDirectory = await privateDirectory(path.join(options.privateRoot, 'output')); const beforeConvert = await validateProviderExecutable(options.executable); diff --git a/cli-connection-reader/model-reader.mjs b/cli-connection-reader/model-reader.mjs index f38c08639..75d26bac0 100644 --- a/cli-connection-reader/model-reader.mjs +++ b/cli-connection-reader/model-reader.mjs @@ -3,7 +3,7 @@ import os from 'node:os'; import path from 'node:path'; import { assertSha256, buildCanonicalRequest, ModelReaderError, providerFingerprintSha256, - lowerableLimits, requestSha256, sha256, + lowerableLimits, READER_SCHEMA_VERSION_V1, READER_SCHEMA_VERSION_V2, requestSha256, sha256, } from './model-contract.mjs'; import { normalizeRevitGlb, parseGlb } from './revit-glb.mjs'; import { normalizeRevitMetadata } from './revit-metadata.mjs'; @@ -116,6 +116,14 @@ function requestLimits(args, deps) { catch (error) { readerError('reference-limits-invalid', 'request', 'Model reader limits are invalid.', false, error); } } +function requestedReaderSchemaVersion(args) { + const version = args['reader-schema-version'] ?? READER_SCHEMA_VERSION_V1; + if (![READER_SCHEMA_VERSION_V1, READER_SCHEMA_VERSION_V2].includes(version)) { + readerError('reference-request-invalid', 'request', 'The requested model-reader schema version is unsupported.'); + } + return version; +} + function validateRequest(command, args, deps) { const limits = requestLimits(args, deps); // This constructs and canonicalizes the complete request-only contract without touching the @@ -126,6 +134,8 @@ function validateRequest(command, args, deps) { limits, protocolVersion: args['expected-provider-protocol'] ?? '1', conversionSettings: args['conversion-settings'] ?? {}, + readerSchemaVersion: requestedReaderSchemaVersion(args), + propertyExpansionLimits: args['property-expansion-limits'] ?? {}, })); } catch (error) { if (error instanceof ModelReaderError) throw error; @@ -172,6 +182,7 @@ async function providerReadiness(args, deps, config, expectedProviderSha256, sig expectedProtocolVersion: args['expected-provider-protocol'] ?? '1', expectedDestination: args['expected-provider-destination'], authorityStorePath: args['authority-store-path'], + readerSchemaVersion: requestedReaderSchemaVersion(args), }); return { ...signing, provider, @@ -218,10 +229,14 @@ async function convertAndCache(args, deps, config, readiness) { const initial = await hashSource(sourcePath, deps.limits); const sourceSha256 = exactExpectedSource(args, initial.sha256); const expectedProtocolVersion = args['expected-provider-protocol'] ?? '1'; + const readerSchemaVersion = requestedReaderSchemaVersion(args); + const propertyExpansionLimits = args['property-expansion-limits'] ?? {}; const canonicalRequest = buildCanonicalRequest({ limits: deps.limits, protocolVersion: expectedProtocolVersion, conversionSettings: args['conversion-settings'] ?? {}, + readerSchemaVersion, + propertyExpansionLimits, }); const identity = { sourceSha256, canonicalRequest, providerFingerprint: readiness.provider.fingerprint, @@ -265,10 +280,24 @@ async function convertAndCache(args, deps, config, readiness) { expectedProtocolVersion, expectedDestination: args['expected-provider-destination'], authorityStorePath: args['authority-store-path'], + readerSchemaVersion, + propertyExpansionLimits, }); emit(deps, 'normalize'); const geometry = normalizeRevitGlb(conversion.outputs.geometry.bytes, { limits: deps.limits }); - const metadata = normalizeRevitMetadata(conversion.outputs.metadata.bytes, geometry.parts, { limits: deps.limits }); + const expectedMetadataSchema = readerSchemaVersion === READER_SCHEMA_VERSION_V2 ? '2' : '1'; + const metadata = normalizeRevitMetadata(conversion.outputs.metadata.bytes, geometry.parts, { + limits: deps.limits, + propertyExpansionLimits: canonicalRequest.propertyExpansionLimits, + expectedSchemaVersion: expectedMetadataSchema, + }); + // The normalizer refuses a mismatch up front, so this is defence in depth + // against that guard being weakened. Assert on the coverage the normalizer + // already returned rather than re-parsing propertiesBytes, which can reach + // maxCanonicalPropertyBytes (16 MB default, 32 MB hard) on every read. + if ((metadata.coverage.metadataSchemaVersion ?? '1') !== expectedMetadataSchema) { + readerError('reference-metadata-invalid', 'normalize-metadata', 'Provider metadata does not match the requested reader schema version.'); + } const finalSource = await hashSource(sourcePath, deps.limits); if (finalSource.sha256 !== sourceSha256) readerError('reference-source-changed', 'source', 'The RVT source changed during conversion.'); const artifacts = { @@ -329,6 +358,8 @@ async function findCachedConversion(args, deps, config, signing, expectedProvide limits: deps.limits, protocolVersion: args['expected-provider-protocol'] ?? '1', conversionSettings: args['conversion-settings'] ?? {}, + readerSchemaVersion: requestedReaderSchemaVersion(args), + propertyExpansionLimits: args['property-expansion-limits'] ?? {}, }); if (!deps.hostAcquireLock || !deps.hostReleaseLock) readerError('reference-provider-host-unavailable', 'cache', 'The managed cache fence is unavailable.'); const withMaintenanceFence = async (work) => { @@ -362,7 +393,7 @@ function summary(result, limits) { const coverage = result.cache.manifest.coverage; const bounds = canonicalGeometryBounds(result.cache.artifacts['geometry.glb'], limits); return { - schemaVersion: 'model-reference-reader/v1', cache: result.hit ? 'hit' : 'miss', + schemaVersion: result.cache.manifest.identity.canonicalRequest.readerSchemaVersion, cache: result.hit ? 'hit' : 'miss', sourceSha256: result.cache.manifest.identity.sourceSha256, canonicalRequestSha256: result.cache.manifest.canonicalRequestSha256, providerFingerprint: result.cache.manifest.identity.providerFingerprint, @@ -413,7 +444,7 @@ export async function runModelCommand(command, args = {}, deps = {}) { if (command === 'preflight') { const readiness = await providerReadiness(args, executionDeps, config, pin, signing); return { - schemaVersion: 'model-reference-reader/v1', ready: true, execution: readiness.provider.describe.execution, + schemaVersion: requestedReaderSchemaVersion(args), ready: true, execution: readiness.provider.describe.execution, provider: readiness.provider.describe, providerFingerprint: readiness.provider.fingerprint, providerFingerprintSha256: readiness.providerFingerprintSha256, signerFingerprintSha256: readiness.signerFingerprintSha256, diff --git a/cli-connection-reader/model-reader.test.mjs b/cli-connection-reader/model-reader.test.mjs index e4c46287d..e6586bc99 100644 --- a/cli-connection-reader/model-reader.test.mjs +++ b/cli-connection-reader/model-reader.test.mjs @@ -114,7 +114,6 @@ test('preflight describes provider and key readiness without conversion or sourc assert.equal(sha256(Buffer.from(out.signerPublicKeyBase64, 'base64')), out.signerFingerprintSha256); assert.deepEqual(state.calls, ['describe']); }); - test('preflight enforces the managed authority-store contract before provider launch', async (t) => { const state = await setup(t); const base = { @@ -226,6 +225,36 @@ test('read-model publishes five binary-safe artifacts with reconciled coverage a assert.equal('sourceArtifactPreimage' in out, false); }); +test('reader v2 binds expansion limits and publishes tagged provider-display property artifacts', async (t) => { + const state = await setup(t); + const versionArgs = { + 'reader-schema-version': 'model-reference-reader/v2', + 'property-expansion-limits': { maxExpandedPropertyRows: 100, maxCanonicalPropertyBytes: 4096 }, + }; + const preflight = await runModelCommand('preflight', { + 'provider-path': state.executable, 'signing-secret-path': state.secretPath, + 'signing-public-path': state.publicPath, ...versionArgs, + }, state.deps); + assert.equal(preflight.schemaVersion, 'model-reference-reader/v2'); + const out = await runModelCommand('read-snapshot', { + ...state.args, ...versionArgs, + 'expected-provider-sha256': preflight.providerFingerprintSha256, + 'expected-signer-sha256': preflight.signerFingerprintSha256, + }, state.deps); + assert.equal(out.schemaVersion, 'model-reference-reader/v2'); + assert.equal(out.coverage.expandedProperties, 1); + assert.deepEqual(out.coverage.effectivePropertyLimits, versionArgs['property-expansion-limits']); + assert.equal(out.packageConfiguration.schemaVersion, 'model-reference-package-configuration/v2'); + const properties = JSON.parse(await fs.readFile(path.join(state.deps.artifactDirectory, out.artifacts.properties.id), 'utf8')); + assert.equal(properties.schemaVersion, '2'); + assert.deepEqual(properties.properties[0], { + entityId: 'element:1001', groupId: 'parameter-group:1', groupName: 'Identity Data', groupOrdinal: 0, + parameterId: 'parameter:1', parameterOrdinal: 0, name: 'Display Mark', unit: null, + valueEncoding: 'provider-display', valueType: 'string', value: 'A-1', + }); + assert.equal(JSON.stringify(out).includes('A-1'), false, 'summary and receipts must not leak property values'); +}); + test('read-snapshot derives public source and package envelopes after private cache verification', async (t) => { const state = await setup(t); const preflight = await runModelCommand('preflight', { diff --git a/cli-connection-reader/model-snapshot.mjs b/cli-connection-reader/model-snapshot.mjs index afda91130..df73c6c96 100644 --- a/cli-connection-reader/model-snapshot.mjs +++ b/cli-connection-reader/model-snapshot.mjs @@ -1,12 +1,14 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { - canonicalJsonBytes, lowerableLimits, ModelReaderError, parseJsonStrict, sha256, + canonicalJsonBytes, lowerableLimits, lowerablePropertyExpansionLimits, ModelReaderError, parseJsonStrict, sha256, } from './model-contract.mjs'; import { signArtifactPreimage } from './model-artifact-auth.mjs'; export const SOURCE_ARTIFACT_DOMAIN = 'AWARE\0model-reference-reader\0source-artifact-set\0v1\0'; export const PACKAGE_ARTIFACT_DOMAIN = 'AWARE\0model-reference-reader\0package-set\0v1\0'; +export const SOURCE_ARTIFACT_DOMAIN_V2 = 'AWARE\0model-reference-reader\0source-artifact-set\0v2\0'; +export const PACKAGE_ARTIFACT_DOMAIN_V2 = 'AWARE\0model-reference-reader\0package-set\0v2\0'; const SOURCE_ORDER = [ ['geometry', 'geometry.glb', 'model/gltf-binary'], @@ -29,11 +31,11 @@ function snapshotError(code, message, details = undefined) { throw new ModelReaderError(code, 'package', false, message, details); } -function document(bytes, key, limits) { +function document(bytes, key, limits, expectedSchemaVersion) { let parsed; try { parsed = parseJsonStrict(bytes, { maxBytes: limits.maxComponentJsonBytes, maxDepth: limits.maxJsonDepth }); } catch (error) { snapshotError('reference-snapshot-source-invalid', `${key} source artifact is invalid.`, error); } - if (parsed?.schemaVersion !== '1' || !Array.isArray(parsed[key])) { + if (parsed?.schemaVersion !== expectedSchemaVersion || !Array.isArray(parsed[key])) { snapshotError('reference-snapshot-source-invalid', `${key} source artifact has an invalid closed document.`); } return parsed; @@ -54,30 +56,36 @@ function receipts(order, bytesByName, itemsByName) { }); } -function packageConfiguration(limits) { +function packageConfiguration(limits, v2, propertyExpansionLimits) { return { - schemaVersion: 'model-reference-package-configuration/v1', + schemaVersion: `model-reference-package-configuration/v${v2 ? '2' : '1'}`, partitionPolicy: 'single-canonical-glb-v1', canonicalOrdering: 'utf8-byte-order-v1', tileBoundaryDuplication: 'none', maximumTileBytes: limits.maxCanonicalGlbBytes, maximumTileTriangles: limits.maxIndices, maximumShardBytes: limits.maxComponentJsonBytes, - maximumShardRecords: Math.max(limits.maxEntities, limits.maxParameters, limits.maxRelationships), + // v2 property rows are bounded by the request's own expansion limits, not by + // the entity/parameter/relationship ceilings. Omitting them let a valid + // document publish more property records than its SIGNED configuration + // claimed, so downstream validation rejected reader-produced output. + maximumShardRecords: Math.max(limits.maxEntities, limits.maxParameters, limits.maxRelationships, + ...(v2 ? [propertyExpansionLimits.maxExpandedPropertyRows] : [])), maximumPackageArtifacts: 6, maximumAggregateBytes: limits.maxCanonicalGlbBytes + (limits.maxComponentJsonBytes * 5), supportedGlb: { version: '2.0', extensions: [], componentTypes: [5121, 5123, 5125, 5126] }, schemas: { - manifest: 'floless.model-snapshot-package/v1', - entities: '1', properties: '1', relationships: '1', index: 'floless.model-snapshot-index/v1', + manifest: `floless.model-snapshot-package/v${v2 ? '2' : '1'}`, + entities: v2 ? '2' : '1', properties: v2 ? 'aware.model-properties/v2' : '1', + relationships: v2 ? '2' : '1', index: `floless.model-snapshot-index/v${v2 ? '2' : '1'}`, }, }; } -function packagedBytes(result, parsed, sourceArtifactEnvelope, configuration) { +function packagedBytes(result, parsed, sourceArtifactEnvelope, configuration, v2) { const entities = parsed.entities.entities; const index = canonicalJsonBytes({ - schemaVersion: 'floless.model-snapshot-index/v1', + schemaVersion: `floless.model-snapshot-index/v${v2 ? '2' : '1'}`, entities: entities.map((entity, ordinal) => ({ id: entity.id, ordinal, tiles: Array.isArray(entity.geometry) && entity.geometry.length > 0 ? ['tile-000000'] : [], })), @@ -97,7 +105,7 @@ function packagedBytes(result, parsed, sourceArtifactEnvelope, configuration) { index: entities.length, }); const manifest = canonicalJsonBytes({ - schemaVersion: 'floless.model-snapshot-package/v1', + schemaVersion: `floless.model-snapshot-package/v${v2 ? '2' : '1'}`, sourceArtifactEnvelopeSha256: sha256(canonicalJsonBytes(sourceArtifactEnvelope)), configurationSha256: sha256(canonicalJsonBytes(configuration)), frame: result.cache.manifest.frame, @@ -149,10 +157,16 @@ async function publishArtifacts(result, directory, sourceBytes, packageBytes) { export async function buildAndPublishSnapshot(result, signingKey, artifactDirectory, options = {}) { if (!result.cache.receiptSha256) snapshotError('reference-cache-authentication-missing', 'The private cache receipt was not authenticated.'); const limits = lowerableLimits(options.limits); + const readerSchemaVersion = result.cache.manifest.identity?.canonicalRequest?.readerSchemaVersion ?? 'model-reference-reader/v1'; + const v2 = readerSchemaVersion === 'model-reference-reader/v2'; + if (!v2 && readerSchemaVersion !== 'model-reference-reader/v1') { + snapshotError('reference-snapshot-source-invalid', 'The reader schema version is unsupported.'); + } + const artifactSchemaVersion = v2 ? '2' : '1'; const parsed = { - entities: document(result.cache.artifacts['entities.json'], 'entities', limits), - properties: document(result.cache.artifacts['properties.json'], 'properties', limits), - relationships: document(result.cache.artifacts['relationships.json'], 'relationships', limits), + entities: document(result.cache.artifacts['entities.json'], 'entities', limits, artifactSchemaVersion), + properties: document(result.cache.artifacts['properties.json'], 'properties', limits, artifactSchemaVersion), + relationships: document(result.cache.artifacts['relationships.json'], 'relationships', limits, artifactSchemaVersion), }; const sourceReceipts = receipts(SOURCE_ORDER, result.cache.artifacts, { geometry: 1, entities: sourceItems('entities', parsed), properties: sourceItems('properties', parsed), @@ -160,7 +174,7 @@ export async function buildAndPublishSnapshot(result, signingKey, artifactDirect }); const identity = result.cache.manifest.identity; const sourceArtifactPreimage = { - schemaVersion: '1', + schemaVersion: artifactSchemaVersion, source: { sourceSha256: identity.sourceSha256, canonicalRequestSha256: result.cache.manifest.canonicalRequestSha256, @@ -170,10 +184,11 @@ export async function buildAndPublishSnapshot(result, signingKey, artifactDirect }, outputs: sourceReceipts, }; - const sourceArtifactEnvelope = signArtifactPreimage(SOURCE_ARTIFACT_DOMAIN, sourceArtifactPreimage, signingKey); - const configuration = packageConfiguration(limits); + const sourceArtifactEnvelope = signArtifactPreimage(v2 ? SOURCE_ARTIFACT_DOMAIN_V2 : SOURCE_ARTIFACT_DOMAIN, sourceArtifactPreimage, signingKey); + const configuration = packageConfiguration(limits, v2, + lowerablePropertyExpansionLimits(result.cache.manifest.identity?.canonicalRequest?.propertyExpansionLimits ?? {})); const configurationSha256 = sha256(canonicalJsonBytes(configuration)); - const packageBytes = packagedBytes(result, parsed, sourceArtifactEnvelope, configuration); + const packageBytes = packagedBytes(result, parsed, sourceArtifactEnvelope, configuration, v2); const aggregateBytes = Object.values(packageBytes).reduce((sum, bytes) => sum + bytes.length, 0); if (aggregateBytes > configuration.maximumAggregateBytes) { snapshotError('reference-output-too-large', 'Snapshot package exceeds its aggregate byte limit.'); @@ -184,7 +199,7 @@ export async function buildAndPublishSnapshot(result, signingKey, artifactDirect 'relationships-000000': parsed.relationships.relationships.length, index: parsed.entities.entities.length, }); const packagePreimage = { - schemaVersion: '1', + schemaVersion: artifactSchemaVersion, source: { sourceArtifactPreimageSha256: sourceArtifactEnvelope.preimageSha256, sourceArtifactEnvelopeSha256: sha256(canonicalJsonBytes(sourceArtifactEnvelope)), @@ -195,12 +210,12 @@ export async function buildAndPublishSnapshot(result, signingKey, artifactDirect signerFingerprintSha256: identity.signerFingerprintSha256, }, packager: { - agent: 'model-reference-reader', version: '0.4.0', - bridgeBuildId: 'aware-connection-reader@0.2.0', configurationSha256, + agent: 'model-reference-reader', version: v2 ? '0.5.0' : '0.4.0', + bridgeBuildId: v2 ? 'aware-connection-reader@0.3.0' : 'aware-connection-reader@0.2.0', configurationSha256, }, outputs: packageReceipts, }; - const packageArtifactEnvelope = signArtifactPreimage(PACKAGE_ARTIFACT_DOMAIN, packagePreimage, signingKey); + const packageArtifactEnvelope = signArtifactPreimage(v2 ? PACKAGE_ARTIFACT_DOMAIN_V2 : PACKAGE_ARTIFACT_DOMAIN, packagePreimage, signingKey); const descriptors = await publishArtifacts(result, artifactDirectory, result.cache.artifacts, packageBytes); return { ...descriptors, sourceArtifactPreimage, sourceArtifactEnvelope, diff --git a/cli-connection-reader/model-snapshot.test.mjs b/cli-connection-reader/model-snapshot.test.mjs index 9fbc4458e..636748e67 100644 --- a/cli-connection-reader/model-snapshot.test.mjs +++ b/cli-connection-reader/model-snapshot.test.mjs @@ -47,3 +47,84 @@ test('snapshot parsing honors the configured component limit above the strict pa configurationSha256: output.packagePreimage.packager.configurationSha256, }); }); + +test('reader v2 publishes independently versioned package schemas and authentication preimages', async (t) => { + const root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), 'aware-model-snapshot-v2-')); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { privateKey, publicKey } = generateKeyPairSync('ed25519'); + const publicKeyBytes = publicKey.export({ format: 'der', type: 'spki' }).subarray(-32); + const artifacts = { + 'geometry.glb': Buffer.from([0x67, 0x6c, 0x54, 0x46]), + 'entities.json': canonicalJsonBytes({ schemaVersion: '2', entities: [] }), + 'properties.json': canonicalJsonBytes({ schemaVersion: '2', properties: [] }), + 'relationships.json': canonicalJsonBytes({ schemaVersion: '2', relationships: [] }), + 'manifest.json': canonicalJsonBytes({ schemaVersion: 'model-reference-manifest/v1' }), + }; + const result = { + key: '6'.repeat(64), + cache: { + receiptSha256: '7'.repeat(64), artifacts, + manifest: { + identity: { + sourceSha256: '8'.repeat(64), signerFingerprintSha256: sha256(publicKeyBytes), + canonicalRequest: { readerSchemaVersion: 'model-reference-reader/v2' }, + }, + canonicalRequestSha256: '9'.repeat(64), providerFingerprintSha256: 'a'.repeat(64), + frame: { units: 'mm', up: 'z', handedness: 'right', axes: ['x', 'y', 'z'] }, + coverage: { unclaimedGeometryNodes: [] }, + }, + }, + }; + const output = await buildAndPublishSnapshot(result, { privateKey, publicKeyBytes }, path.join(root, 'artifacts')); + assert.equal(output.sourceArtifactPreimage.schemaVersion, '2'); + assert.equal(output.packagePreimage.schemaVersion, '2'); + assert.equal(output.packageConfiguration.schemaVersion, 'model-reference-package-configuration/v2'); + assert.equal(output.packageConfiguration.schemas.manifest, 'floless.model-snapshot-package/v2'); + assert.equal(output.packageConfiguration.schemas.properties, 'aware.model-properties/v2'); + assert.equal(output.packagePreimage.packager.version, '0.5.0'); + const manifest = JSON.parse(await fs.readFile(path.join(root, 'artifacts', output.packageArtifacts.manifest.id), 'utf8')); + assert.equal(manifest.schemaVersion, 'floless.model-snapshot-package/v2'); +}); + +test('a v2 shard configuration advertises the property expansion ceiling it must cover', async (t) => { + // v2 property rows are bounded by propertyExpansionLimits, not by the + // entity/parameter/relationship ceilings. Signing a configuration that omits + // them advertised fewer records than a valid document may legitimately + // publish, so downstream package validation rejected reader-produced output. + const root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), 'aware-model-snapshot-v2-limits-')); + t.after(() => fs.rm(root, { recursive: true, force: true })); + const { privateKey, publicKey } = generateKeyPairSync('ed25519'); + const publicKeyBytes = publicKey.export({ format: 'der', type: 'spki' }).subarray(-32); + const artifacts = { + 'geometry.glb': Buffer.from([0x67, 0x6c, 0x54, 0x46]), + 'entities.json': canonicalJsonBytes({ schemaVersion: '2', entities: [] }), + 'properties.json': canonicalJsonBytes({ schemaVersion: '2', properties: [] }), + 'relationships.json': canonicalJsonBytes({ schemaVersion: '2', relationships: [] }), + 'manifest.json': canonicalJsonBytes({ schemaVersion: 'model-reference-manifest/v1' }), + }; + const result = { + key: '6'.repeat(64), + cache: { + receiptSha256: '7'.repeat(64), artifacts, + manifest: { + identity: { + sourceSha256: '8'.repeat(64), signerFingerprintSha256: sha256(publicKeyBytes), + canonicalRequest: { + readerSchemaVersion: 'model-reference-reader/v2', + // Lowered model limits, a deliberately HIGHER expansion ceiling. + propertyExpansionLimits: { maxExpandedPropertyRows: 4096 }, + }, + }, + canonicalRequestSha256: '9'.repeat(64), providerFingerprintSha256: 'a'.repeat(64), + frame: { units: 'mm', up: 'z', handedness: 'right', axes: ['x', 'y', 'z'] }, + coverage: { unclaimedGeometryNodes: [] }, + }, + }, + }; + // Lowered model ceilings, a deliberately higher property expansion ceiling: + // exactly the shape where the two disagree. + const output = await buildAndPublishSnapshot(result, { privateKey, publicKeyBytes }, path.join(root, 'artifacts'), + { limits: { maxEntities: 2, maxParameters: 2, maxRelationships: 2 } }); + assert.ok(output.packageConfiguration.maximumShardRecords >= 4096, + `signed maximumShardRecords ${output.packageConfiguration.maximumShardRecords} is below the expansion ceiling it must cover`); +}); diff --git a/cli-connection-reader/model-windows-harness.mjs b/cli-connection-reader/model-windows-harness.mjs index f9529320c..bf2697eb5 100644 --- a/cli-connection-reader/model-windows-harness.mjs +++ b/cli-connection-reader/model-windows-harness.mjs @@ -110,7 +110,15 @@ try { executable: provider, executableSha256: sha256(readFileSync(provider)), operation: 'describe', cwd: unrelatedCwd, environment: { LANG: 'C', LC_ALL: 'C', TZ: 'UTC' }, stdin: Buffer.from('{}'), - timeoutMs: 10_000, stdoutLimit: 1024 * 1024, stderrLimit: 1024 * 1024, + // First execution of a freshly written .exe, and this job now runs it after + // two compiler/SDK closure copies, a native vendor-repro build, cargo build + // and SEA staging -- so the launch pays a cold Defender scan on a runner + // that is already worked hard, and 10s expired before the provider could + // reach its abort (exit 124 instead of the expected 134). The assertion + // below is what makes this test mean something; the budget only decides how + // long a genuine hang takes to surface, so give the launch the same order + // of room this file's own run() default allows. + timeoutMs: 60_000, stdoutLimit: 1024 * 1024, stderrLimit: 1024 * 1024, }); assert.equal(legacy.exitCode, 134, 'the legacy case-sensitive provider environment must reproduce the Node SEA CSPRNG abort'); } finally { @@ -150,12 +158,12 @@ try { const appDirectory = path.join(temporary, 'rvt-reader-e2e'); mkdirSync(appDirectory); const appSource = path.join(appDirectory, 'rvt-reader-e2e.flo'); writeFileSync(appSource, `app: rvt-reader-e2e -version: 0.4.0 +version: 0.5.0 display-name: RVT Reader E2E description: Exercise the authenticated local RVT reader through a real one-shot AWARE app. exposes-as-agent: false requires: - - model-reference-reader@0.4.0 + - model-reference-reader@0.5.0 requires-permissions: filesystem: - read: '*.rvt' diff --git a/cli-connection-reader/package-lock.json b/cli-connection-reader/package-lock.json index b0d87cdd9..0b9a74ebe 100644 --- a/cli-connection-reader/package-lock.json +++ b/cli-connection-reader/package-lock.json @@ -1,12 +1,12 @@ { "name": "aware-connection-reader", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "aware-connection-reader", - "version": "0.2.0", + "version": "0.3.0", "dependencies": { "fflate": "^0.8.3", "web-ifc": "0.0.77" diff --git a/cli-connection-reader/package.json b/cli-connection-reader/package.json index 7117dde73..fde2bc11f 100644 --- a/cli-connection-reader/package.json +++ b/cli-connection-reader/package.json @@ -1,6 +1,6 @@ { "name": "aware-connection-reader", - "version": "0.2.0", + "version": "0.3.0", "private": true, "type": "module", "description": "AWARE cli-transport bridge: extract steel connections from an IFC as tessellated mesh scene primitives (drives web-ifc WASM).", @@ -9,6 +9,7 @@ }, "scripts": { "build": "node build.mjs", + "verify:repro": "node verify-reproducible-builds.mjs", "test": "node --test", "test:windows-harness": "node model-windows-harness.mjs" }, diff --git a/cli-connection-reader/repro-settings.mjs b/cli-connection-reader/repro-settings.mjs new file mode 100644 index 000000000..0d4c22e5c --- /dev/null +++ b/cli-connection-reader/repro-settings.mjs @@ -0,0 +1,8 @@ +export const READER_BUILD_SETTINGS = Object.freeze({ + bundle: Object.freeze({ platform: 'node', format: 'cjs', target: 'node24' }), + sea: Object.freeze({ + disableExperimentalWarning: true, + section: 'NODE_SEA_BLOB', + sentinelFuse: 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2', + }), +}); diff --git a/cli-connection-reader/revit-metadata.mjs b/cli-connection-reader/revit-metadata.mjs index 951243664..b7d82ac33 100644 --- a/cli-connection-reader/revit-metadata.mjs +++ b/cli-connection-reader/revit-metadata.mjs @@ -1,4 +1,7 @@ -import { assertClosedObject, canonicalJsonBytes, lowerableLimits, ModelReaderError, parseJsonStrict, sha256 } from './model-contract.mjs'; +import { + assertClosedObject, canonicalJsonBytes, lowerableLimits, lowerablePropertyExpansionLimits, + ModelReaderError, parseJsonStrict, sha256, +} from './model-contract.mjs'; const POSITIVE_INT64 = /^(?:[1-9]\d*)$/; const SIGNED_INT64 = /^(?:0|-?[1-9]\d*)$/; @@ -58,8 +61,12 @@ function tableIndex(value, table, label, nullable = false) { return table[value]; } -function validateParameter(parameter, index) { - closed(parameter, ['id', 'name', 'unit', 'readable', 'storageType', 'value'], [], `parameters[${index}]`); +function validateSourceStorageParameter(parameter, index, tagged = false) { + const required = tagged + ? ['id', 'name', 'unit', 'valueEncoding', 'readable', 'storageType', 'value'] + : ['id', 'name', 'unit', 'readable', 'storageType', 'value']; + closed(parameter, required, [], `parameters[${index}]`); + if (tagged && parameter.valueEncoding !== 'source-storage') invalid(`parameters[${index}].valueEncoding must be source-storage`); const id = positiveId(parameter.id, `parameters[${index}].id`); const name = text(parameter.name, `parameters[${index}].name`); const unit = text(parameter.unit, `parameters[${index}].unit`, true); @@ -83,9 +90,32 @@ function validateParameter(parameter, index) { value = signedId(value, `parameters[${index}].value`); if (!parameter.readable) invalid('element-id parameter must be readable'); } else invalid(`parameters[${index}] has unsupported storageType`); - return { id, name, unit, readable: parameter.readable, storageType, value }; + return { id, name, unit, ...(tagged ? { valueEncoding: 'source-storage' } : {}), readable: parameter.readable, storageType, value }; } +function validateV2Parameter(parameter, index) { + if (parameter?.valueEncoding === 'source-storage') return validateSourceStorageParameter(parameter, index, true); + closed(parameter, ['id', 'name', 'unit', 'valueEncoding', 'valueType', 'value'], [], `parameters[${index}]`); + if (parameter.valueEncoding !== 'provider-display') invalid(`parameters[${index}].valueEncoding is unsupported`); + const id = positiveId(parameter.id, `parameters[${index}].id`); + const name = text(parameter.name, `parameters[${index}].name`); + const unit = text(parameter.unit, `parameters[${index}].unit`, true); + if (!['string', 'number'].includes(parameter.valueType)) invalid(`parameters[${index}].valueType is unsupported`); + let value = parameter.value; + if (parameter.valueType === 'string') value = text(value, `parameters[${index}].value`); + else { + if (typeof value !== 'number' || !Number.isFinite(value)) invalid(`parameters[${index}].value must be a finite number`); + value = Object.is(value, -0) ? 0 : value; + } + return { id, name, unit, valueEncoding: 'provider-display', valueType: parameter.valueType, value }; +} + +function assertUniqueReferences(refs, label) { + if (new Set(refs).size !== refs.length) invalid(`${label} contains duplicate references`); +} + +const propertyDocumentBaseBytes = Buffer.byteLength('{"properties":[],"schemaVersion":"2"}', 'utf8'); + function bounds(parts) { const positions = parts.flatMap((part) => part.positions ?? []); if (!positions.length) return null; @@ -143,6 +173,7 @@ function assertAcyclic(relations, kind) { export function normalizeRevitMetadata(input, geometryParts, options = {}) { const limits = lowerableLimits(options.limits); + const propertyLimits = lowerablePropertyExpansionLimits(options.propertyExpansionLimits); let metadata = input; if (typeof input === 'string' || Buffer.isBuffer(input) || input instanceof Uint8Array) { try { metadata = parseJsonStrict(input, { maxBytes: limits.maxMetadataBytes, maxDepth: limits.maxJsonDepth }); } @@ -150,7 +181,15 @@ export function normalizeRevitMetadata(input, geometryParts, options = {}) { } closed(metadata, ['schemaVersion', 'document', 'types', 'levels', 'parameterGroups', 'parameters', 'elements', 'relations'], [], 'metadata'); - if (metadata.schemaVersion !== '1') invalid('unsupported metadata schemaVersion'); + if (!['1', '2'].includes(metadata.schemaVersion)) invalid('unsupported metadata schemaVersion'); + const metadataV2 = metadata.schemaVersion === '2'; + // Enforce the REQUESTED schema before anything is expanded. Checked after the + // fact, a v1 response to a v2 request normalizes under v1's far larger + // expansion allowance, so a compact document can allocate millions of + // property rows before the mismatch is noticed. + if (options.expectedSchemaVersion !== undefined && metadata.schemaVersion !== options.expectedSchemaVersion) { + invalid('Provider metadata does not match the requested reader schema version.'); + } closed(metadata.document, ['kind', 'id'], [], 'document'); if (metadata.document.kind !== 'revit-project' || typeof metadata.document.id !== 'string' || !metadata.document.id) invalid('document must be an identified Revit project'); @@ -159,10 +198,18 @@ export function normalizeRevitMetadata(input, geometryParts, options = {}) { if (level.elevation !== undefined && (typeof level.elevation !== 'number' || !Number.isFinite(level.elevation))) invalid(`levels[${index}].elevation must be finite`); return level; }); - const parameters = list(metadata.parameters, 'parameters', limits.maxParameters).map(validateParameter); + const parameters = list(metadata.parameters, 'parameters', limits.maxParameters) + .map(metadataV2 ? validateV2Parameter : (parameter, index) => validateSourceStorageParameter(parameter, index, false)); if (new Set(parameters.map((entry) => entry.id)).size !== parameters.length) invalid('parameters contains duplicate ids'); + if (metadataV2) parameters.forEach((parameter, index) => { + if (parameter.id !== String(index + 1)) invalid(`parameters[${index}].id must equal its one-based table index`); + }); const parameterGroups = uniqueTable(metadata.parameterGroups, 'parameterGroups', limits.maxParameters, ['parameters']).map((group, index) => { const refs = list(group.parameters, `parameterGroups[${index}].parameters`, limits.maxParameters); + if (metadataV2) { + if (group.id !== String(index + 1)) invalid(`parameterGroups[${index}].id must equal its one-based table index`); + assertUniqueReferences(refs, `parameterGroups[${index}].parameters`); + } return { ...group, parameters: refs.map((value, ordinal) => tableIndex(value, parameters, `parameterGroups[${index}].parameters[${ordinal}]`)) }; }); @@ -179,6 +226,10 @@ export function normalizeRevitMetadata(input, geometryParts, options = {}) { const elementIds = new Set(); const owners = new Map(); const propertyRows = []; + const reachedGroups = new Set(); + const reachedParameters = new Set(); + let elementGroupReferences = 0; + let canonicalPropertyBytes = propertyDocumentBaseBytes; const entities = rawElements.map((element, elementOrdinal) => { closed(element, ['id', 'revitClass', 'category', 'family', 'type', 'level', 'parameterGroups', 'appearances'], ['ifcGuid'], `elements[${elementOrdinal}]`); @@ -195,26 +246,48 @@ export function normalizeRevitMetadata(input, geometryParts, options = {}) { owners.set(name, id); joined.push({ nodeName: name, parts: parts.map((part) => part.primitiveOrdinal ?? 0) }); } - const groups = list(element.parameterGroups, `elements[${elementOrdinal}].parameterGroups`, limits.maxParameters) + const groupRefs = list(element.parameterGroups, `elements[${elementOrdinal}].parameterGroups`, limits.maxParameters); + if (metadataV2) assertUniqueReferences(groupRefs, `elements[${elementOrdinal}].parameterGroups`); + elementGroupReferences += groupRefs.length; + const groups = groupRefs .map((value, ordinal) => tableIndex(value, parameterGroups, `elements[${elementOrdinal}].parameterGroups[${ordinal}]`)); const guidValues = []; - groups.forEach((group, groupOrdinal) => group.parameters.forEach((parameter, parameterOrdinal) => { - if (propertyRows.length >= limits.maxParameters) invalid('expanded property count exceeds its limit', 'reference-output-too-large'); - propertyRows.push({ - entityId: `element:${id}`, - groupId: `parameter-group:${group.id}`, - groupName: group.name, - groupOrdinal, - parameterId: `parameter:${parameter.id}`, - parameterOrdinal, - name: parameter.name, - unit: parameter.unit, - readable: parameter.readable, - storageType: parameter.storageType, - value: parameter.value, + groups.forEach((group, groupOrdinal) => { + reachedGroups.add(group.id); + group.parameters.forEach((parameter, parameterOrdinal) => { + reachedParameters.add(parameter.id); + const row = { + entityId: `element:${id}`, + groupId: `parameter-group:${group.id}`, + groupName: group.name, + groupOrdinal, + parameterId: `parameter:${parameter.id}`, + parameterOrdinal, + name: parameter.name, + unit: parameter.unit, + ...(metadataV2 + ? (parameter.valueEncoding === 'provider-display' + ? { valueEncoding: 'provider-display', valueType: parameter.valueType } + : { valueEncoding: 'source-storage', readable: parameter.readable, storageType: parameter.storageType }) + : { readable: parameter.readable, storageType: parameter.storageType }), + value: parameter.value, + }; + if (metadataV2) { + if (propertyRows.length >= propertyLimits.maxExpandedPropertyRows) invalid('expanded property count exceeds its limit', 'reference-output-too-large'); + const separatorBytes = propertyRows.length > 0 ? 1 : 0; + const nextBytes = canonicalPropertyBytes + separatorBytes + canonicalJsonBytes(row).length; + if (!Number.isSafeInteger(nextBytes) || nextBytes > propertyLimits.maxCanonicalPropertyBytes) { + invalid('canonical property artifact exceeds its byte limit', 'reference-output-too-large'); + } + canonicalPropertyBytes = nextBytes; + } else if (propertyRows.length >= limits.maxParameters) { + invalid('expanded property count exceeds its limit', 'reference-output-too-large'); + } + propertyRows.push(row); + if (parameter.valueEncoding !== 'provider-display' && parameter.name === 'IfcGUID' + && parameter.storageType === 'string' && parameter.readable && parameter.value) guidValues.push(parameter.value); }); - if (parameter.name === 'IfcGUID' && parameter.storageType === 'string' && parameter.readable && parameter.value) guidValues.push(parameter.value); - })); + }); if (new Set(guidValues).size > 1) invalid(`element ${id} has conflicting IfcGUID parameters`); const ifcGuid = guidValues[0] ?? null; if (element.ifcGuid !== undefined && element.ifcGuid !== ifcGuid) invalid(`element ${id} redundant ifcGuid does not match its authoritative parameter`); @@ -267,9 +340,11 @@ export function normalizeRevitMetadata(input, geometryParts, options = {}) { relationships.sort((a, b) => Buffer.compare(Buffer.from(a.kind), Buffer.from(b.kind)) || compareDecimal(a.from, b.from) || compareDecimal(a.to, b.to) || compareDecimal(a.id, b.id)); const unclaimedGeometryNodes = [...geometryByName.keys()].filter((name) => !owners.has(name)).sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b))); - const entitiesBytes = canonicalJsonBytes({ schemaVersion: '1', entities }); - const propertiesBytes = canonicalJsonBytes({ schemaVersion: '1', properties: propertyRows }); - const relationshipsBytes = canonicalJsonBytes({ schemaVersion: '1', relationships }); + const artifactSchemaVersion = metadataV2 ? '2' : '1'; + const entitiesBytes = canonicalJsonBytes({ schemaVersion: artifactSchemaVersion, entities }); + const propertiesBytes = canonicalJsonBytes({ schemaVersion: artifactSchemaVersion, properties: propertyRows }); + const relationshipsBytes = canonicalJsonBytes({ schemaVersion: artifactSchemaVersion, relationships }); + if (metadataV2 && propertiesBytes.length !== canonicalPropertyBytes) invalid('canonical property byte accounting mismatch'); for (const [label, bytes] of [['entities', entitiesBytes], ['properties', propertiesBytes], ['relationships', relationshipsBytes]]) { if (bytes.length > limits.maxComponentJsonBytes) invalid(`${label} artifact exceeds its byte limit`, 'reference-output-too-large'); } @@ -283,6 +358,17 @@ export function normalizeRevitMetadata(input, geometryParts, options = {}) { unclaimedGeometryNodes, entitySetSha256: digestIds(entities.map((entity) => entity.id)), geometryNodeSetSha256: digestIds(geometryByName.keys()), + ...(metadataV2 ? { + metadataSchemaVersion: '2', + nativeParameterGroups: parameterGroups.length, + nativeParameters: parameters.length, + elementGroupReferences, + expandedProperties: propertyRows.length, + orphanParameterGroups: parameterGroups.length - reachedGroups.size, + orphanParameters: parameters.length - reachedParameters.size, + canonicalPropertyBytes: propertiesBytes.length, + effectivePropertyLimits: propertyLimits, + } : {}), }; return { entities, properties: propertyRows, relationships, entitiesBytes, propertiesBytes, relationshipsBytes, coverage }; } diff --git a/cli-connection-reader/revit-metadata.test.mjs b/cli-connection-reader/revit-metadata.test.mjs index 36c557cf6..b3a537136 100644 --- a/cli-connection-reader/revit-metadata.test.mjs +++ b/cli-connection-reader/revit-metadata.test.mjs @@ -8,6 +8,19 @@ const geometry = [ { nodeName: 'part-b', primitiveOrdinal: 0, positions: [[20, 0, 0], [30, 0, 0], [20, 10, 0]], triangles: [[0, 1, 2]] }, ]; +function makeMetadataV2() { + const metadata = makeMetadataFixture(); + metadata.schemaVersion = '2'; + metadata.parameterGroups[0].id = '1'; + metadata.parameters = [ + { id: '1', name: 'IfcGUID', unit: null, valueEncoding: 'provider-display', valueType: 'string', value: 'display-only-guid' }, + { id: '2', name: 'Length', unit: 'mm', valueEncoding: 'provider-display', valueType: 'number', value: -0 }, + ]; + metadata.parameterGroups[0].parameters = [0, 1]; + delete metadata.elements[0].ifcGuid; + return metadata; +} + test('explicit indexed metadata resolves to stable namespaced identity and multipart geometry', () => { const result = normalizeRevitMetadata(makeMetadataFixture({ elementId: '9223372036854775806', nodeNames: ['part-a', 'part-b'] }), geometry); assert.equal(result.entities[0].id, 'element:9223372036854775806'); @@ -35,6 +48,80 @@ test('parameter storage types preserve signed element ids, null, empty, boolean assert.deepEqual(result.properties.map((row) => row.value), [null, true, '-12', 1.25, '', '-2000011']); }); +test('v2 preserves provider-display provenance, normalizes negative zero, and never derives IfcGUID identity', () => { + const result = normalizeRevitMetadata(makeMetadataV2(), geometry.slice(0, 1)); + assert.equal(JSON.parse(result.propertiesBytes).schemaVersion, '2'); + assert.deepEqual(result.properties.map((row) => ({ + valueEncoding: row.valueEncoding, valueType: row.valueType, value: row.value, unit: row.unit, + })), [ + { valueEncoding: 'provider-display', valueType: 'string', value: 'display-only-guid', unit: null }, + { valueEncoding: 'provider-display', valueType: 'number', value: 0, unit: 'mm' }, + ]); + assert.equal(result.entities[0].ifcGuid, null); + assert.deepEqual({ + metadataSchemaVersion: result.coverage.metadataSchemaVersion, + nativeParameterGroups: result.coverage.nativeParameterGroups, + nativeParameters: result.coverage.nativeParameters, + elementGroupReferences: result.coverage.elementGroupReferences, + expandedProperties: result.coverage.expandedProperties, + orphanParameterGroups: result.coverage.orphanParameterGroups, + orphanParameters: result.coverage.orphanParameters, + canonicalPropertyBytes: result.coverage.canonicalPropertyBytes, + effectivePropertyLimits: result.coverage.effectivePropertyLimits, + }, { + metadataSchemaVersion: '2', nativeParameterGroups: 1, nativeParameters: 2, + elementGroupReferences: 1, expandedProperties: 2, orphanParameterGroups: 0, orphanParameters: 0, + canonicalPropertyBytes: result.propertiesBytes.length, + effectivePropertyLimits: { maxExpandedPropertyRows: 100_000, maxCanonicalPropertyBytes: 16 * 1024 * 1024 }, + }); +}); + +test('v2 source-storage rows retain authoritative semantics and alone may supply IfcGUID identity', () => { + const metadata = makeMetadataV2(); + metadata.parameters[0] = { + id: '1', name: 'IfcGUID', unit: null, valueEncoding: 'source-storage', + readable: true, storageType: 'string', value: 'authoritative-guid', + }; + const result = normalizeRevitMetadata(metadata, geometry.slice(0, 1)); + assert.equal(result.entities[0].ifcGuid, 'authoritative-guid'); + assert.deepEqual(result.properties[0], { + entityId: 'element:1001', groupId: 'parameter-group:1', groupName: 'Identity Data', groupOrdinal: 0, + parameterId: 'parameter:1', parameterOrdinal: 0, name: 'IfcGUID', unit: null, + valueEncoding: 'source-storage', readable: true, storageType: 'string', value: 'authoritative-guid', + }); +}); + +test('v2 rejects duplicate references and reports unreachable table rows without expanding them', () => { + const duplicateGroup = makeMetadataV2(); + duplicateGroup.elements[0].parameterGroups = [0, 0]; + assert.throws(() => normalizeRevitMetadata(duplicateGroup, geometry.slice(0, 1)), /duplicate references/); + + const duplicateParameter = makeMetadataV2(); + duplicateParameter.parameterGroups[0].parameters = [0, 0]; + assert.throws(() => normalizeRevitMetadata(duplicateParameter, geometry.slice(0, 1)), /duplicate references/); + + const orphaned = makeMetadataV2(); + orphaned.parameterGroups.push({ id: '2', name: 'Unused group', parameters: [] }); + orphaned.parameters.push({ id: '3', name: 'Unused', unit: null, valueEncoding: 'provider-display', valueType: 'string', value: 'unused' }); + const result = normalizeRevitMetadata(orphaned, geometry.slice(0, 1)); + assert.equal(result.coverage.orphanParameterGroups, 1); + assert.equal(result.coverage.orphanParameters, 1); + assert.equal(result.properties.length, 2); +}); + +test('v2 enforces independent pre-append row and canonical property byte ceilings', () => { + const metadata = makeMetadataV2(); + assert.throws(() => normalizeRevitMetadata(metadata, geometry.slice(0, 1), { + propertyExpansionLimits: { maxExpandedPropertyRows: 1 }, + }), (error) => error.code === 'reference-output-too-large'); + assert.throws(() => normalizeRevitMetadata(metadata, geometry.slice(0, 1), { + propertyExpansionLimits: { maxCanonicalPropertyBytes: 64 }, + }), (error) => error.code === 'reference-output-too-large'); + assert.throws(() => normalizeRevitMetadata(metadata, geometry.slice(0, 1), { + propertyExpansionLimits: { maxExpandedPropertyRows: 2_000_001 }, + }), /hard ceiling/); +}); + test('explicit relations validate endpoints, provider kinds, acyclic parents, and canonical order', () => { const metadata = makeMetadataFixture({ elementId: '2', nodeNames: ['part-b'] }); const firstElement = { ...metadata.elements[0], id: '1', appearances: ['part-a'] }; @@ -139,3 +226,20 @@ test('aggregate property expansion is bounded before repeated references allocat (error) => error.code === 'reference-output-too-large', ); }); + +test('a schema mismatch is refused before the requested limits can be bypassed', () => { + // A v1 response to a v2 request used to be normalized first and compared + // after, so it expanded under v1's far larger allowance. The mismatch must be + // refused up front: the error is the schema mismatch, NOT an expansion + // ceiling, which is what proves nothing was expanded first. + const v1 = makeMetadataFixture(); + const tight = { maxExpandedPropertyRows: 1, maxCanonicalPropertyBytes: 1 }; + assert.throws( + () => normalizeRevitMetadata(v1, geometry, { propertyExpansionLimits: tight, expectedSchemaVersion: '2' }), + (error) => error.code === 'reference-metadata-invalid' + && /does not match the requested reader schema version/.test(error.message)); + // The matching request still normalizes, so the guard is not simply refusing everything. + assert.ok(normalizeRevitMetadata(v1, geometry, { expectedSchemaVersion: '1' })); + // Absent an expectation the normalizer keeps its existing both-schemas contract. + assert.ok(normalizeRevitMetadata(v1, geometry, {})); +}); diff --git a/cli-connection-reader/test-fixtures/model-provider-fixture.mjs b/cli-connection-reader/test-fixtures/model-provider-fixture.mjs index 91d2bc207..3394e3718 100644 --- a/cli-connection-reader/test-fixtures/model-provider-fixture.mjs +++ b/cli-connection-reader/test-fixtures/model-provider-fixture.mjs @@ -31,7 +31,17 @@ async function main() { } : {}; await fs.writeFile(geometryPath, makeGlbFixture(geometryOptions)); - await fs.writeFile(metadataPath, JSON.stringify(makeMetadataFixture())); + const metadata = makeMetadataFixture(); + if (request.canonicalRequest?.readerSchemaVersion === 'model-reference-reader/v2') { + metadata.schemaVersion = '2'; + metadata.parameterGroups[0].id = '1'; + metadata.parameters = [{ + id: '1', name: 'Display Mark', unit: null, + valueEncoding: 'provider-display', valueType: 'string', value: 'A-1', + }]; + delete metadata.elements[0].ifcGuid; + } + await fs.writeFile(metadataPath, JSON.stringify(metadata)); process.stdout.write(JSON.stringify({ ...provenance, documentKind: 'revit-project', sourceSha256: request.sourceSha256, geometryPath, metadataPath, diff --git a/cli-connection-reader/verify-reproducible-builds.mjs b/cli-connection-reader/verify-reproducible-builds.mjs new file mode 100644 index 000000000..6560b05b4 --- /dev/null +++ b/cli-connection-reader/verify-reproducible-builds.mjs @@ -0,0 +1,92 @@ +#!/usr/bin/env node +import { createHash } from 'node:crypto'; +import { lstatSync, readFileSync, readdirSync, realpathSync } from 'node:fs'; +import { join, relative, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const portable = (path) => path.split(sep).join('/'); +const sha256 = (bytes) => createHash('sha256').update(bytes).digest('hex'); + +function files(root, path = root) { + const stat = lstatSync(path); + if (stat.isSymbolicLink()) throw new Error(`reproducibility output contains symbolic link: ${path}`); + if (stat.isFile()) return [path]; + if (!stat.isDirectory()) throw new Error(`unsupported reproducibility output entry: ${path}`); + return readdirSync(path, { withFileTypes: true }) + .flatMap((entry) => files(root, join(path, entry.name))); +} + +export function inventory(root) { + const absolute = resolve(root); + return files(absolute).map((path) => ({ + path: portable(relative(absolute, path)), + size: lstatSync(path).size, + sha256: sha256(readFileSync(path)), + })).sort((a, b) => Buffer.compare(Buffer.from(a.path), Buffer.from(b.path))); +} + +export function forbiddenEncodings(root) { + const variants = new Set(); + const add = (value) => { + if (!value) return; + // Case folding and escaping compose. A serialized Windows path with altered + // casing (c:\\users\\alice) is BOTH lowercased and JSON-escaped, and deriving + // each transform only from the raw spelling never produced that combination, + // so a forbidden root could survive this proof. Fold first, then escape. + for (const separator of [value, value.replaceAll('\\', '/')]) { + for (const cased of [separator, separator.toLowerCase()]) { + variants.add(cased); + variants.add(JSON.stringify(cased).slice(1, -1)); + try { variants.add(new URL(`file:///${cased.replaceAll('\\', '/')}`).href); } catch { /* invalid path is still checked raw */ } + } + } + }; + add(resolve(root)); + return [...variants].flatMap((value) => [ + { label: value, bytes: Buffer.from(value, 'utf8') }, + { label: `${value} (UTF-16LE)`, bytes: Buffer.from(value, 'utf16le') }, + ]).filter((entry) => entry.bytes.length); +} + +export function scanForbiddenRoots(outputRoot, forbiddenRoots) { + const needles = forbiddenRoots.flatMap(forbiddenEncodings); + const hits = []; + for (const path of files(resolve(outputRoot))) { + const bytes = readFileSync(path); + const lowerUtf8 = Buffer.from(bytes.toString('utf8').toLowerCase(), 'utf8'); + const lowerUtf16 = Buffer.from(bytes.toString('utf16le').toLowerCase(), 'utf16le'); + for (const needle of needles) { + if (bytes.includes(needle.bytes) || lowerUtf8.includes(needle.bytes) || lowerUtf16.includes(needle.bytes)) { + hits.push({ path: portable(relative(resolve(outputRoot), path)), rootEncoding: needle.label }); + } + } + } + return hits; +} + +export function verifyReproducibleOutputs({ left, right, forbiddenRoots = [] }) { + const leftInventory = inventory(left); const rightInventory = inventory(right); + if (JSON.stringify(leftInventory) !== JSON.stringify(rightInventory)) { + throw new Error(`builder outputs differ:\nleft=${JSON.stringify(leftInventory, null, 2)}\nright=${JSON.stringify(rightInventory, null, 2)}`); + } + const hits = [...scanForbiddenRoots(left, forbiddenRoots), ...scanForbiddenRoots(right, forbiddenRoots)]; + if (hits.length) throw new Error(`builder root leaked into compared artifact: ${JSON.stringify(hits)}`); + return leftInventory; +} + +function parseArgs(argv) { + const out = { forbiddenRoots: [] }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; const value = argv[index + 1]; + if (arg === '--forbid-root' && value != null) { out.forbiddenRoots.push(value); index += 1; continue; } + if ((arg === '--left' || arg === '--right') && value != null) { out[arg.slice(2)] = value; index += 1; continue; } + throw new Error('arguments are --left DIR --right DIR [--forbid-root DIR ...]'); + } + if (!out.left || !out.right) throw new Error('arguments are --left DIR --right DIR [--forbid-root DIR ...]'); + return out; +} + +if (realpathSync(fileURLToPath(import.meta.url)) === realpathSync(process.argv[1])) { + const result = verifyReproducibleOutputs(parseArgs(process.argv.slice(2))); + console.log(`reproducible output proof passed: ${result.length} byte-identical files`); +} diff --git a/cli-connection-reader/verify-reproducible-builds.test.mjs b/cli-connection-reader/verify-reproducible-builds.test.mjs new file mode 100644 index 000000000..f4d96439b --- /dev/null +++ b/cli-connection-reader/verify-reproducible-builds.test.mjs @@ -0,0 +1,48 @@ +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { resolve } from 'node:path'; +import { forbiddenEncodings, verifyReproducibleOutputs } from './verify-reproducible-builds.mjs'; + +function roots() { + const root = mkdtempSync(join(tmpdir(), 'aware-repro-compare-')); + const left = join(root, 'left-short'); const right = join(root, 'right-much-longer'); + mkdirSync(left); mkdirSync(right); + return { root, left, right }; +} + +test('two closed ordered inventories must match by path, size, and hash', () => { + const { left, right } = roots(); + writeFileSync(join(left, 'aware.exe'), 'same'); writeFileSync(join(right, 'aware.exe'), 'same'); + assert.deepEqual(verifyReproducibleOutputs({ left, right }), [{ + path: 'aware.exe', size: 4, + sha256: '0967115f2813a3541eaef77de9d9d5773f1c0c04314b0bbfe4ff3b3b1c55b5d5', + }]); + writeFileSync(join(right, 'aware.exe'), 'different'); + assert.throws(() => verifyReproducibleOutputs({ left, right }), /builder outputs differ/); +}); + +test('checkout roots are rejected in raw, normalized, JSON, URI, case, and UTF-16 forms', () => { + const { root, left, right } = roots(); + writeFileSync(join(left, 'receipt.json'), Buffer.from(root.toUpperCase(), 'utf16le')); + writeFileSync(join(right, 'receipt.json'), Buffer.from(root.toUpperCase(), 'utf16le')); + assert.throws(() => verifyReproducibleOutputs({ left, right, forbiddenRoots: [root] }), /root leaked/); +}); + +test('a root that is BOTH case-folded and JSON-escaped is a needle on every platform', () => { + // Derived from a SYNTHETIC Windows root, not from tmpdir(): this suite runs on + // ubuntu in CI, where a real temp path has no backslashes, so + // JSON.stringify(root) is the identity and the needle collapses to + // root.toLowerCase() -- which the unfixed code already emitted. The literal + // backslashes below survive resolve() on POSIX as ordinary characters, so the + // fold-then-escape combination is genuinely absent before the fix everywhere. + const root = 'C:\\Users\\Alice\\src\\aware'; + const labels = new Set(forbiddenEncodings(root).map((entry) => entry.label)); + const escapedFolded = JSON.stringify(resolve(root).toLowerCase()).slice(1, -1); + assert.ok(labels.has(escapedFolded), `missing lowercased JSON-escaped needle: ${escapedFolded}`); + // The separately-generated forms must still be there. + assert.ok(labels.has(resolve(root).toLowerCase()), 'missing folded needle'); + assert.ok(labels.has(JSON.stringify(resolve(root)).slice(1, -1)), 'missing escaped needle'); +}); diff --git a/cli/PRIVATE-WINDOWS-BUILD.md b/cli/PRIVATE-WINDOWS-BUILD.md new file mode 100644 index 000000000..9e1386d2a --- /dev/null +++ b/cli/PRIVATE-WINDOWS-BUILD.md @@ -0,0 +1,129 @@ +# Private Windows runtime build inputs + +This is unsigned build evidence, not a releasable installer. Signing, trusted builder +isolation, independent approvals, reservations and promotion remain separate gates. + +`create-windows-internal-repro-inputs.mjs` accepts one local JSON input with exactly +`schema`, `source`, `inputs`, `tools` and `closures`. Its schema is +`aware-windows-repro-builder-inputs/v1`. `source` contains the exact Git `commit`, +`tree` and absolute source `bundle` path. Use canonical Git blob bytes for the six +input files; a checkout with line-ending conversions is not equivalent evidence. + +| Input ID | File in that exact source | +| --- | --- | +| aware-cargo-lock | cli/Cargo.lock | +| reader-package-lock | cli-connection-reader/package-lock.json | +| builder-script | cli/build-windows-internal-repro.mjs | +| compiler-closure-script | cli/windows-compiler-closure.mjs | +| reader-settings-script | cli-connection-reader/repro-settings.mjs | +| compiler-audit-script | cli/windows-compiler-audit.ps1 | + +`tools` has exactly six absolute files: `git`, `node`, `npm-cli`, `postject`, +`web-ifc-wasm`, and `powershell`. Node is Windows x64 24.14.0. PowerShell must be the +canonical inbox Windows PowerShell under the loader-observed System32 directory; +PowerShell 7 and copied hosts are not accepted. The builder hashes its executable +helpers, extracts the source with the authenticated Git tool and inline OS bootstrap, +then matches the helpers to those Git blobs before evaluation. JavaScript helpers +are imported from authenticated in-memory bytes; PowerShell receives the exact +authenticated script bytes over stdin, rather than re-reading a mutable script path. +Legacy `Add-Type` compilation uses a fresh, access-restricted ASCII directory +under the loader-observed Windows `Temp` directory. That parent must be canonical, +ASCII and writable; otherwise bootstrap refuses. The helper removes only its own +scratch directory, including after compilation failure. Compiler-child paths and +TEMP/TMP remain the declared private paths; no machine locale or installed files +are changed. + +`closures` has exactly these eleven absolute directories. Inventory every file in +each selected component. Do not prune DLLs, headers or libraries to fit a disk quota. + +| Closure ID | Selected installed component | +| --- | --- | +| npm-cache | Sealed offline npm cache | +| cargo-home | Sealed Cargo source directory containing vendor/ | +| compiler-rust-bin | Direct Rust 1.95.0 toolchain bin/; never rustup shims | +| compiler-rust-lib | The same toolchain lib/, including rustlib/ | +| compiler-msvc-bin | MSVC bin/Hostx64/x64/ | +| compiler-msvc-include | That MSVC version's include/ | +| compiler-msvc-lib | That MSVC version's lib/x64/ | +| compiler-sdk-include | Windows SDK Include/VERSION/ | +| compiler-sdk-um-lib | Windows SDK Lib/VERSION/um/x64/ | +| compiler-sdk-ucrt-lib | Windows SDK Lib/VERSION/ucrt/x64/ | +| compiler-sdk-bin | Windows SDK bin/VERSION/x64/ | + +The factory emits a canonical path-free manifest and a local-only locator. The +locator has no environment or separate compiler executable paths. Roots must be +local absolute Windows drive paths of at most 200 characters, without redirection +links, ambiguous path components, controls, semicolons or equals signs. Spaces and +Unicode are supported. Retain local evidence privately: it contains physical paths. + +Launch the exact digest-bound builder with the exact Node executable and an explicit +native Windows environment containing only the OS-derived `SystemRoot`. Node's +Windows child launcher can add PATH even with `env: {}`; that launch is refused. +A native ProcessStartInfo launcher must clear EnvironmentVariables, supply only +SystemRoot, preserve exact argv/cwd, and concurrently drain both redirected streams. +Do not forward the Visual Studio developer shell environment. PATH, INCLUDE, LIB, +LIBPATH and tool roles are constructed from fresh verified private copies. Cargo +and Rust receive descriptor-owned `VCINSTALLDIR` pointing at the private MSVC +directory and `VSCMD_ARG_TGT_ARCH=x64`, so MSVC lookup uses this fixed environment +instead of loading the machine's Visual Studio discovery DLL. Cargo +uses the private vendor copy and a separate empty CARGO_HOME. Existing targets, +network fallback, shared mutable compiler inputs and byte patching are not admitted. +Private npm logs are directed outside its copied cache. The cache inventory must +still match after consumption before any runtime receipt can be issued. + +The Windows debugger audit observes compiler descendants and their loaded images +inside an owned non-breakaway job. Its process count must reconcile with the job, +and all images must be declared private inputs, derived Cargo outputs or files in +the explicitly protected Windows locations. It never attaches to unrelated work. +Pre/post inventories and private Rust sysroot checks are mandatory. + +Every v3 audit records its effective, fixed startup policy, including an explicit +null when the MSVC inventory has no `vctip.exe`. When that exact private file is +created as a descendant, the auditor verifies its size/hash while the creation +event is paused, stops only that process before its entry point, and observes its +real exit with code `0xe0000488`. All process counts, image checks and zero-active +completion requirements still apply. No telemetry file is omitted or changed, no +machine policy is changed, and an unrelated process is never targeted. The native +Cargo fixture explicitly starts the copied helper from a build script to prove +this branch even when the linker does not request telemetry itself. + +Process lifetime ordinals distinguish legitimate PID reuse from overlapping live +processes. Creation, DLL and exit events bind to that lifetime, and root identity +never follows a reused numeric PID. The event counter includes every observed debug +event; retained lifetime/image events are ordered within it, while thread, exception, +unload and debug-string events need not retain their payloads. Exited lifetimes lose +their active handle/breakpoint state. Windows owns closure of debugger-provided +handles when EXIT is continued; see Microsoft's [debugging-event contract](https://learn.microsoft.com/en-us/windows/win32/debug/debugging-events). +The v2 schema/policy is not accepted by new-source provenance validation. + +Each exclusive request retains raw stdout/stderr, a combined command log and a +launch sidecar with status, signal, timeout budget and spawn error details, even +when PowerShell itself fails. Capture files refuse replacement; all writes are +attempted and execution/persistence errors are preserved together. An incomplete +audit remains diagnostic evidence and can never authorize successful provenance. + +Before comparing A/B artifact inventories, call `verifyCompilerProvenance` from +the reviewed compiler helper separately for both output roots and retain the returned +evidence digests in the comparison report. This checks source/build identity, all +five required complete process audits, compiler inputs, successful exits and the +exact artifact inventory. Raw audit files stay outside the byte-compared payload. +Recheck both npm/Cargo sources and scan artifacts for original/private physical roots. + +The required Windows native gate is `node cli/windows-vendor-repro.native.mjs`. +It discovers installed inputs only while constructing test manifests, copies its own +original inputs, hides those originals, and executes the production private compiler +path. It checks Rust and C bytes, SDK/CRT provenance, lib/rc/rustdoc, private-mutation +refusals and vendor-remap red controls. `AWARE_REPRO_COMPILER_ROOTS` may name a local +JSON mapping of the nine component names without the `compiler-` prefix to installed +paths; otherwise use a VS x64 developer environment. `AWARE_REPRO_TEST_EVIDENCE` +retains per-side audit files outside the test's unique temporary fixture directory. +The fixture cleans only its own newly created copies, sequentially between sides. +Its paths retain Polish characters and an emoji. Include provenance comes from +UTF-8 `/sourceDependencies` records, and `/LINKREPRO` captures actual library/object +bytes for comparison with the private inventories. Console diagnostic encoding is +not an authority: raw output is retained, while missing, foreign or altered input +records fail verification. + +Reserve at least 10 GB free for a full paired runtime/package run and recheck between +stages. Compression is extra margin, not a capacity guarantee. Do not remove retained +failed-build evidence or weaken closure inventories to make the build fit. diff --git a/cli/build-windows-internal-repro.mjs b/cli/build-windows-internal-repro.mjs new file mode 100644 index 000000000..438b8c5db --- /dev/null +++ b/cli/build-windows-internal-repro.mjs @@ -0,0 +1,559 @@ +#!/usr/bin/env node +// Dedicated private Windows-x64 build boundary. It extracts an immutable Git bundle into a new root, +// admits only digest-bound tools/offline closures, and builds Rust plus the connection-reader without +// consulting a developer checkout, PATH tool, or mutable shared target directory. +import { execFileSync, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { + copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync, rmSync, + writeFileSync, +} from 'node:fs'; +import { basename, dirname, join, relative, resolve, sep, win32 } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const scriptPath = fileURLToPath(import.meta.url); +const SHA256 = /^[0-9a-f]{64}$/; +const SHA1 = /^[0-9a-f]{40}$/; +const TARGET = 'x86_64-pc-windows-msvc'; +export const COMMAND_OUTPUT_BUFFER_BYTES = 128 * 1024 * 1024; +export const SOURCE_PATHS = Object.freeze(['cli', 'cli-connection-reader']); +const LOGICAL_ROOTS = ['', '', '', '']; +export const WINDOWS_LOGICAL_RUST_FLAGS = ['-C', 'link-arg=/Brepro', + ...LOGICAL_ROOTS.flatMap((root) => Array(2).fill(`--remap-path-prefix=${root}=${root}`)), +].join(' '); +export const WINDOWS_LOGICAL_NATIVE_FLAGS = '/Brepro /experimental:deterministic "/pathmap:=" "/pathmap:="'; +export const LOGICAL_CARGO_COMMAND = ' build --manifest-path /cli/Cargo.toml --release --locked --offline --config source.crates-io.replace-with="vendored-sources" --config source.vendored-sources.directory="" --verbose --verbose'; +const EXACT_POISON = new Set([ + 'RUSTFLAGS', 'RUSTDOCFLAGS', 'CARGO_ENCODED_RUSTFLAGS', 'CC', 'AR', 'CFLAGS', 'CL', 'LINK', 'LIB', 'INCLUDE', + 'PATH', 'LIBPATH', '_CL_', 'RUSTC', 'RUSTDOC', 'VSLANG', 'VCINSTALLDIR', 'VCTOOLSINSTALLDIR', 'VSINSTALLDIR', 'SDKROOT', + 'NODE_OPTIONS', 'ESBUILD_BINARY_PATH', 'GOOGLE_CLIENT_SECRET', 'AWARE_GOOGLE_CLIENT_SECRET', '_NO_DEBUG_HEAP', +]); +const POISON_PREFIXES = ['npm_config_', 'DOTNET_', 'COREHOST_', 'COMPLUS_', 'CARGO_', 'RUSTUP_', + 'VSCMD_', 'WINDOWSSDK', 'SDK_', 'CC_', 'AR_', 'CFLAGS_', 'RUSTC_', 'RUSTDOC_']; +const CODE_INPUT_IDS = ['builder-script', 'compiler-closure-script', 'reader-settings-script', 'compiler-audit-script']; +export function runningInputFiles(builderScript = scriptPath) { + const cli = dirname(builderScript); + return { 'builder-script': builderScript, 'compiler-closure-script': join(cli, 'windows-compiler-closure.mjs'), + 'reader-settings-script': join(cli, '..', 'cli-connection-reader', 'repro-settings.mjs'), + 'compiler-audit-script': join(cli, 'windows-compiler-audit.ps1') }; +} +export function verifyBootstrapInputs(inputs, files = runningInputFiles()) { + const bytes = {}; + for (const id of CODE_INPUT_IDS) { + if (!SHA256.test(inputs?.[id] ?? '') || !existsSync(files[id])) { + throw new Error(`running ${id} differs from its manifest authority`); + } + bytes[id] = readFileSync(files[id]); + if (sha256Bytes(bytes[id]) !== inputs[id]) throw new Error(`running ${id} differs from its manifest authority`); + } + return bytes; +} +export async function loadVerifiedBuildModules(inputs, files = runningInputFiles(), sourceRoot) { + const bytes = verifyBootstrapInputs(inputs, files); + if (!sourceRoot) throw new Error('extracted source must be authenticated before helper evaluation'); + verifyExtractedInputs(sourceRoot, inputs, files['builder-script']); + const compiler = await import(`data:text/javascript;base64,${bytes['compiler-closure-script'].toString('base64')}`); + const settings = await import(`data:text/javascript;base64,${bytes['reader-settings-script'].toString('base64')}`); + return { compiler, settings: settings.READER_BUILD_SETTINGS }; +} + +// These small validators are deliberately inline: no unauthenticated helper or +// rejected network/device path may be evaluated or dereferenced during bootstrap. +export function validateBootstrapPath(path, label) { + if (typeof path !== 'string' || !/^[a-z]:[\\/]/i.test(path) || path.length > 200 + || /[;=<>"|?*\x00-\x1f\x7f]/.test(path) || path.slice(2).includes(':') + || path.slice(3).split(/[\\/]/).some(part => !part || part === '.' || part === '..' || /[. ]$/.test(part) + || /^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part))) throw new Error(`unsafe bootstrap ${label} path`); +} +export function validateBootstrapLocator(locator) { + if (!locator || canonicalJson(Object.keys(locator).sort()) !== canonicalJson(['closures', 'schema', 'sourceBundle', 'tools'])) throw new Error('invalid bootstrap locator keys'); + if (locator.schema !== 'aware-windows-repro-locator/v1') throw new Error('unsupported bootstrap locator schema'); + const toolIds = ['git', 'node', 'npm-cli', 'postject', 'web-ifc-wasm', 'powershell']; + const closureIds = ['npm-cache', 'cargo-home', ...['rust-bin', 'rust-lib', 'msvc-bin', 'msvc-include', 'msvc-lib', 'sdk-include', 'sdk-um-lib', 'sdk-ucrt-lib', 'sdk-bin'].map(id => `compiler-${id}`)]; + for (const [kind, ids] of [['tools', toolIds], ['closures', closureIds]]) { + if (!locator[kind] || canonicalJson(Object.keys(locator[kind]).sort()) !== canonicalJson([...ids].sort())) throw new Error(`invalid bootstrap ${kind}`); + for (const id of ids) validateBootstrapPath(locator[kind][id], id); + } + validateBootstrapPath(locator.sourceBundle, 'source bundle'); +} +export function bootstrapSystemEnvironment(tempRoot, sharedObjects = process.report.getReport().sharedObjects) { + const dirs = ['kernel32.dll', 'ntdll.dll'].map(name => { + const matches = [...new Set(sharedObjects.filter(path => win32.basename(path).toLowerCase() === name).map(path => win32.dirname(path).toLowerCase()))]; + if (matches.length !== 1 || win32.basename(matches[0]) !== 'system32') throw new Error(`invalid loader-observed ${name}`); + return matches[0]; + }); + if (dirs[0] !== dirs[1]) throw new Error('loader-observed Windows modules disagree'); + const system32 = dirs[0], windows = win32.dirname(system32); + return { SystemRoot: windows, WINDIR: windows, ComSpec: win32.join(system32, 'cmd.exe'), PATH: system32, + PATHEXT: '.COM;.EXE;.BAT;.CMD', TEMP: tempRoot, TMP: tempRoot }; +} + +const sha256Bytes = (bytes) => createHash('sha256').update(bytes).digest('hex'); +const sha256File = (path) => sha256Bytes(readFileSync(path)); +const canonical = (value) => Array.isArray(value) ? value.map(canonical) + : value && typeof value === 'object' + ? Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])) : value; +const canonicalJson = (value) => `${JSON.stringify(canonical(value), null, 2)}\n`; +const portable = (path) => path.split(sep).join('/'); + +export function writeBuilderManifestEvidence({ artifactsRoot, manifestText }) { + if (canonicalJson(JSON.parse(manifestText)) !== manifestText) { + throw new Error('builder manifest evidence must already be canonical JSON'); + } + const path = join(artifactsRoot, 'builder-manifest.json'); + writeFileSync(path, manifestText, { encoding: 'utf8', flag: 'wx' }); + return { size: lstatSync(path).size, sha256: sha256File(path) }; +} + +export function rejectedAmbientKeys(env) { + const counts = Object.keys(env).reduce((map, key) => map.set(key.toLowerCase(), (map.get(key.toLowerCase()) ?? 0) + 1), new Map()); + return Object.keys(env).filter((key) => counts.get(key.toLowerCase()) > 1 || EXACT_POISON.has(key.toUpperCase()) + || POISON_PREFIXES.some((prefix) => key.toLowerCase().startsWith(prefix.toLowerCase()))) + .sort(); +} + +export function closedGitEnvironment(systemEnv) { + return { + ...systemEnv, + GIT_ALLOW_PROTOCOL: 'file', + GIT_CONFIG_COUNT: '0', + GIT_CONFIG_GLOBAL: 'NUL', + GIT_CONFIG_NOSYSTEM: '1', + GIT_TERMINAL_PROMPT: '0', + }; +} + +function verifyFileRecord(id, manifest, locator) { + const record = manifest.tools?.[id]; const path = locator.tools?.[id]; + if (!record || record.id !== id || !SHA256.test(record.sha256 ?? '')) throw new Error(`invalid tool record: ${id}`); + if (typeof path !== 'string' || !existsSync(path) || !lstatSync(path).isFile()) throw new Error(`missing tool path: ${id}`); + const actual = sha256File(path); + if (actual !== record.sha256) throw new Error(`tool digest mismatch for ${id}: ${actual}`); + return resolve(path); +} + +function inventory(root) { + const walk = (path) => readdirSync(path, { withFileTypes: true }).flatMap((entry) => { + const child = join(path, entry.name); + if (entry.isSymbolicLink()) throw new Error(`offline closure contains symbolic link: ${child}`); + if (entry.isDirectory()) return walk(child); + if (!entry.isFile()) throw new Error(`offline closure contains unsupported entry: ${child}`); + return [child]; + }); + return walk(root).map((path) => ({ + path: portable(relative(root, path)), size: lstatSync(path).size, sha256: sha256File(path), + })).sort((a, b) => Buffer.compare(Buffer.from(a.path), Buffer.from(b.path))); +} + +function verifyClosure(id, manifest, locator, inventoryFunction = inventory) { + const root = locator.closures?.[id]; const expected = manifest.closures?.[id]; + if (typeof root !== 'string' || !existsSync(root) || !lstatSync(root).isDirectory()) throw new Error(`missing closure: ${id}`); + const actual = inventoryFunction(resolve(root)); + if (canonicalJson(actual) !== canonicalJson(expected?.files)) throw new Error(`offline closure inventory mismatch: ${id}`); + return resolve(root); +} +export function verifyConsumedClosure(id, root, manifest, inventoryFunction = inventory) { + return verifyClosure(id, manifest, { closures: { [id]: root } }, inventoryFunction); +} + +export function materializeClosure(id, source, destination, manifest, inventoryFunction = inventory) { + const files = inventoryFunction(source); + if (canonicalJson(files) !== canonicalJson(manifest.closures?.[id]?.files)) throw new Error(`source offline closure inventory mismatch: ${id}`); + if (existsSync(destination)) throw new Error('private closure destination must be fresh'); + mkdirSync(destination, { recursive: true }); + for (const record of files) { + const output = join(destination, ...record.path.split('/')); mkdirSync(dirname(output), { recursive: true }); + copyFileSync(join(source, ...record.path.split('/')), output); + } + if (canonicalJson(inventoryFunction(destination)) !== canonicalJson(manifest.closures?.[id]?.files)) { + throw new Error(`materialized offline closure inventory mismatch: ${id}`); + } + return destination; +} + +export function cargoArguments(manifestPath, vendorDirectory) { + if (typeof vendorDirectory !== 'string' || !vendorDirectory) throw new Error('Cargo vendor directory is required'); + const vendor = vendorDirectory.replaceAll('\\', '/').replaceAll('"', '\\"'); + return ['build', '--manifest-path', manifestPath, '--release', '--locked', '--offline', + '--config', 'source.crates-io.replace-with="vendored-sources"', + '--config', `source.vendored-sources.directory="${vendor}"`, + '--verbose', '--verbose']; +} + +export function verifiedVendorDirectory(cargoClosure) { + if (typeof cargoClosure !== 'string' || !cargoClosure.trim()) throw new Error('verified Cargo closure is required'); + const vendor = resolve(cargoClosure, 'vendor'); + if (!existsSync(vendor) || !lstatSync(vendor).isDirectory() || lstatSync(vendor).isSymbolicLink()) { + throw new Error('verified Cargo vendor path must be an existing directory'); + } + return vendor; +} + +function pathSpellings(path) { + return [...new Set([path, path.replaceAll('\\', '/')])]; +} + +export function rustCompilerArguments({ workRoot, sourceRoot, cargoHome, cargoVendor }) { + if (typeof cargoVendor !== 'string' || !cargoVendor.trim() + || !existsSync(cargoVendor) || !lstatSync(cargoVendor).isDirectory() || lstatSync(cargoVendor).isSymbolicLink()) { + throw new Error('Cargo vendor path must be an existing directory'); + } + // rustc applies the LAST matching textual prefix: broad roots precede their children. + const roots = [[workRoot, ''], [sourceRoot, ''], [cargoHome, ''], [cargoVendor, '']]; + const remaps = roots.flatMap(([path, token]) => { + if (typeof path !== 'string' || !path || /[\x00\x1f]/.test(path)) throw new Error('invalid compiler remap root'); + return pathSpellings(path).map((spelling) => `--remap-path-prefix=${spelling}=${token}`); + }); + return ['-C', 'link-arg=/Brepro', ...remaps]; +} + +export function normalizeBuildText(text, roots) { + const replacements = roots.flatMap(([path, token]) => pathSpellings(path) + .flatMap((spelling) => [[spelling, token], [spelling.replaceAll('\\', '\\\\'), token]])) + .sort((left, right) => right[0].length - left[0].length); + for (const [path, token] of replacements) { + const escaped = path.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + text = text.replace(new RegExp(escaped, 'gi'), () => token); + } + return text; +} + +export function nativeCompilerEnvironmentFlags(workRoot) { + validateBootstrapPath(workRoot, 'native work root'); + // Every native input is materialized under work. One mapped root avoids MSVC's + // first-match ambiguity and keeps CL below its documented 1024-character limit. + const flags = ['/Brepro', '/experimental:deterministic', + ...pathSpellings(win32.normalize(workRoot)).map(path => `"/pathmap:${path}="`)].join(' '); + if (flags.length > 1024) throw new Error('native CL environment exceeds 1024 characters'); + return flags; +} + +export function verifyRustHostVersion(text) { + const lines = text.trim().split(/\r?\n/); + if (!/^rustc 1\.95\.0\b/.test(lines[0] ?? '') + || lines.filter(line => line.startsWith('host:')).length !== 1 + || !lines.includes(`host: ${TARGET}`)) throw new Error('pinned native Rust version/host mismatch'); +} + +export function rejectSourceCargoConfiguration(sourceRoot, cargoHome) { + const entry = path => { + try { return lstatSync(path); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } + }; + const check = directory => { + if (entry(directory)?.isSymbolicLink()) throw new Error('redirected Cargo configuration is forbidden'); + for (const name of ['config', 'config.toml']) { + // lstat also catches a dangling symlink: unreadable config must not disappear. + if (entry(join(directory, name))) throw new Error('source/ancestor/private Cargo configuration is forbidden'); + } + }; + // Cargo discovers configuration all the way to the drive root, not just in its + // extracted cwd. An output parent's build.target would silently drop host flags. + for (let directory = resolve(sourceRoot);;) { + check(join(directory, '.cargo')); + const parent = dirname(directory); if (parent === directory) break; directory = parent; + } + check(join(sourceRoot, 'cli', '.cargo')); check(cargoHome); +} + +export function controlledEnvironment({ compiler, workRoot, sourceRoot, cargoHome, cargoVendor, tempRoot }) { + if (!compiler?.host?.windows || !compiler.host.system32 || !compiler.environment || !compiler.tools) throw new Error('verified private compiler is required'); + return { + SystemRoot: compiler.host.windows, WINDIR: compiler.host.windows, ComSpec: join(compiler.host.system32, 'cmd.exe'), + ...compiler.environment, + TEMP: tempRoot, TMP: tempRoot, + CARGO_HOME: cargoHome, CARGO_TARGET_DIR: join(workRoot, 'cargo-target'), CARGO_NET_OFFLINE: 'true', + CARGO_BUILD_JOBS: '1', + RUSTC: compiler.tools.rustc, RUSTDOC: compiler.tools.rustdoc, + CARGO_ENCODED_RUSTFLAGS: rustCompilerArguments({ workRoot, sourceRoot, cargoHome, cargoVendor }).join('\x1f'), + CARGO_TARGET_X86_64_PC_WINDOWS_MSVC_LINKER: compiler.tools.link, + CC: compiler.tools.cl, AR: join(workRoot, 'cargo-target', 'native-tools', 'aware-lib.exe'), CFLAGS: '/Brepro', CL: nativeCompilerEnvironmentFlags(workRoot), + CC_x86_64_pc_windows_msvc: compiler.tools.cl, AR_x86_64_pc_windows_msvc: join(workRoot, 'cargo-target', 'native-tools', 'aware-lib.exe'), + SOURCE_DATE_EPOCH: '0', TZ: 'UTC', + }; +} + +export function verifyCargoVendorBinding({ args, env, cargoVendor }) { + const operand = `source.vendored-sources.directory="${cargoVendor.replaceAll('\\', '/')}"`; + const vendors = args.filter(arg => arg.startsWith('source.vendored-sources.directory=')); + if (vendors.length !== 1 || vendors[0] !== operand) throw new Error('Cargo vendor operand differs from the verified root'); + const flags = env.CARGO_ENCODED_RUSTFLAGS.split('\x1f'); + const vendorMaps = flags.filter(arg => arg.endsWith('=')); + const expected = pathSpellings(cargoVendor).map(path => `--remap-path-prefix=${path}=`); + if (canonicalJson(vendorMaps) !== canonicalJson(expected)) throw new Error('compiler vendor remaps differ from the verified root'); +} + +export function logicalCargoCommand(args, roots) { + const command = normalizeBuildText(['', ...args].join(' '), roots).replaceAll('\\', '/'); + if (command !== LOGICAL_CARGO_COMMAND) throw new Error('Cargo command differs from the closed build contract'); + return command; +} + +export function createCargoBuild({ compiler, workRoot, sourceRoot, cargoHome, cargoClosure, tempRoot }) { + const cargoVendor = verifiedVendorDirectory(cargoClosure); + const env = Object.freeze(controlledEnvironment({ compiler, workRoot, sourceRoot, cargoHome, cargoVendor, tempRoot })); + const args = Object.freeze(cargoArguments(join(sourceRoot, 'cli', 'Cargo.toml'), cargoVendor)); + verifyCargoVendorBinding({ args, env, cargoVendor }); + const command = logicalCargoCommand(args, [[sourceRoot, ''], [cargoVendor, '']]); + return Object.freeze({ cargoVendor, env, args, command }); +} + +// Decode the Windows argv quoting used by Cargo's verbose command display. A +// missing closing quote is missing proof, never a compilation to silently skip. +function verboseWindowsArguments(command) { + const args = []; let index = 0; + while (index < command.length) { + while (/\s/.test(command[index] ?? '') && index < command.length) index++; + if (index === command.length) break; + let value = '', quoted = false; + while (index < command.length && (quoted || !/\s/.test(command[index]))) { + let slashes = 0; + while (command[index] === '\\') { slashes++; index++; } + if (command[index] === '"') { + value += '\\'.repeat(Math.floor(slashes / 2)); + if (slashes % 2) value += '"'; else quoted = !quoted; + index++; + } else { + value += '\\'.repeat(slashes); + if (index < command.length && (quoted || !/\s/.test(command[index]))) value += command[index++]; + } + } + if (quoted) throw new Error('verbose Cargo proof has an unparseable rustc argument'); + args.push(value); + } + return args; +} + +export function assertVerboseCargoProof(text, rustArgs = [], rustcPath) { + const checks = [ + [/--release/, 'release profile'], [/--locked/, 'locked mode'], [/--offline/, 'offline mode'], + ]; + for (const [pattern, label] of checks) if (!pattern.test(text)) throw new Error(`verbose Cargo proof omitted ${label}`); + const commands = text.split(/\r?\n/).filter(line => /^\s*Running\b/.test(line)) + .flatMap(line => { + const candidate = /--crate-name\b/.test(line) || /(?:^|&&\s*)(?:"[a-z]:[\\/][^"\r\n]*[\\/]rustc\.exe"|[a-z]:[\\/][^"=\r\n]*[\\/]rustc\.exe)(?:\s|`|$)/i.test(line.slice(line.indexOf('`') + 1)); + if (!candidate) return []; + // Cargo's Windows display prints the executable path verbatim even when + // it contains spaces; only its arguments use Windows argv quoting. + const match = /(?:^|&& )(?:("[a-z]:[\\/][^"\r\n]*[\\/]rustc\.exe")|([a-z]:[\\/][^"=\r\n]*[\\/]rustc\.exe)) (--crate-name .*)`$/i.exec(line.slice(line.indexOf('`') + 1)); + if (!/^\s*Running `/.test(line) || !match) throw new Error('verbose Cargo proof has an unparseable rustc compilation'); + const executable = (match[1] ?? match[2]).replaceAll('"', ''); + if (rustcPath && win32.normalize(executable).toLowerCase() !== win32.normalize(rustcPath).toLowerCase()) { + throw new Error('verbose Cargo proof used a different rustc'); + } + return [verboseWindowsArguments(match[3])]; + }); + if (!commands.length) throw new Error('verbose Cargo proof omitted actual rustc compilations'); + for (const command of commands) { + if (!command.some((arg, index) => (arg === '-C' && command[index + 1] === 'link-arg=/Brepro') || arg === '-Clink-arg=/Brepro')) throw new Error('rustc compilation omitted /Brepro'); + for (const argument of rustArgs.filter(arg => arg.startsWith('--remap-path-prefix='))) { + if (!command.includes(argument)) { + throw new Error(`rustc compilation omitted compiler remap: ${argument}`); + } + } + } +} + +export function verifyExtractedInputs(source, inputs, runningScript = scriptPath) { + const running = runningInputFiles(runningScript), extracted = runningInputFiles(join(source, 'cli', 'build-windows-internal-repro.mjs')); + const actual = { + 'aware-cargo-lock': sha256File(join(source, 'cli', 'Cargo.lock')), + 'reader-package-lock': sha256File(join(source, 'cli-connection-reader', 'package-lock.json')), + ...Object.fromEntries(CODE_INPUT_IDS.map(id => [id, sha256File(extracted[id])])), + }; + if (Object.entries(actual).some(([id, digest]) => inputs?.[id] !== digest) + || CODE_INPUT_IDS.some(id => actual[id] !== sha256File(running[id]))) { + throw new Error('extracted source locks or builder script differ from running builder and manifest'); + } +} + +function run(path, args, options = {}) { + const result = spawnSync(path, args, { + encoding: 'utf8', windowsHide: true, maxBuffer: COMMAND_OUTPUT_BUFFER_BYTES, ...options, + }); + const combined = `${result.stdout ?? ''}${result.stderr ?? ''}`; + if (result.error || result.status !== 0) throw new Error(`${basename(path)} failed (${result.status}): ${result.error?.message ?? combined}`); + return combined; +} + +function parseArgs(argv) { + const out = {}; + for (let i = 0; i < argv.length; i += 2) { + if (!argv[i]?.startsWith('--') || argv[i + 1] == null) throw new Error('arguments are --manifest FILE --locator FILE --output DIR'); + out[argv[i].slice(2)] = argv[i + 1]; + } + if (!out.manifest || !out.locator || !out.output) throw new Error('arguments are --manifest FILE --locator FILE --output DIR'); + return out; +} + +export function verifyBuildAuthority({ manifest, locator, env = process.env }) { + validateBootstrapLocator(locator); + if (manifest?.schema !== 'aware-windows-repro-builder/v1') throw new Error('unsupported builder manifest schema'); + if (locator?.schema !== 'aware-windows-repro-locator/v1') throw new Error('unsupported builder locator schema'); + const poisoned = rejectedAmbientKeys(env); + if (poisoned.length) throw new Error(`ambient build authority is forbidden: ${poisoned.join(', ')}`); + if (manifest.platform !== 'win32' || manifest.arch !== 'x64' || manifest.nodeVersion !== '24.14.0' + || manifest.rustVersion !== '1.95.0' || manifest.target !== TARGET) throw new Error('unsupported pinned build platform/toolchain'); + if (process.platform !== 'win32' || process.arch !== 'x64' || process.versions.node !== manifest.nodeVersion) { + throw new Error('wrapper must run under the pinned Windows x64 Node'); + } + const toolIds = ['git', 'node', 'npm-cli', 'postject', 'web-ifc-wasm', 'powershell']; + const tools = Object.fromEntries(toolIds.map((id) => [id, verifyFileRecord(id, manifest, locator)])); + verifyBootstrapInputs(manifest.inputs); + if (realpathSync(tools.node) !== realpathSync(process.execPath)) throw new Error('verified node.exe is not the running Node'); + const bundle = locator.sourceBundle; + if (typeof bundle !== 'string' || !existsSync(bundle) || sha256File(bundle) !== manifest.source?.bundleSha256) { + throw new Error('source bundle digest mismatch'); + } + if (!SHA1.test(manifest.source?.commit ?? '') || !SHA1.test(manifest.source?.tree ?? '')) throw new Error('invalid source commit/tree'); + return { tools, bundle: resolve(bundle) }; +} + +export async function buildWindowsInternal({ manifestPath, locatorPath, outputRoot, env = process.env }) { + for (const [label, path] of Object.entries({ manifestPath, locatorPath, outputRoot })) validateBootstrapPath(path, label); + const manifestText = canonicalJson(JSON.parse(readFileSync(manifestPath, 'utf8'))); + const manifest = JSON.parse(manifestText); const locator = JSON.parse(readFileSync(locatorPath, 'utf8')); + const authority = verifyBuildAuthority({ manifest, locator, env }); + const output = resolve(outputRoot); + if (existsSync(output) && readdirSync(output).length) throw new Error('output root must be absent or empty'); + mkdirSync(output, { recursive: true }); + const work = join(output, 'work'); const artifacts = join(output, 'artifacts'); const evidence = join(output, 'evidence'); + const source = join(work, 'source'); const tempRoot = join(work, 'temp'); + mkdirSync(tempRoot, { recursive: true }); mkdirSync(source); mkdirSync(artifacts); mkdirSync(evidence); + const gitEnv = closedGitEnvironment(bootstrapSystemEnvironment(tempRoot)); + run(authority.tools.git, ['clone', '--no-checkout', '--config', 'core.autocrlf=false', authority.bundle, source], { env: gitEnv }); + run(authority.tools.git, ['sparse-checkout', 'init', '--no-cone'], { cwd: source, env: gitEnv }); + run(authority.tools.git, ['sparse-checkout', 'set', '--no-cone', ...SOURCE_PATHS.map((path) => `/${path}/`)], + { cwd: source, env: gitEnv }); + run(authority.tools.git, ['checkout', '--detach', manifest.source.commit], { cwd: source, env: gitEnv }); + const commit = run(authority.tools.git, ['rev-parse', 'HEAD'], { cwd: source, env: gitEnv }).trim(); + const tree = run(authority.tools.git, ['rev-parse', 'HEAD^{tree}'], { cwd: source, env: gitEnv }).trim(); + if (commit !== manifest.source.commit || tree !== manifest.source.tree) throw new Error('extracted source identity mismatch'); + const unexpectedSource = readdirSync(source).filter((name) => name !== '.git' && !SOURCE_PATHS.includes(name)); + if (unexpectedSource.length) throw new Error(`sparse source boundary contains unexpected roots: ${unexpectedSource.join(', ')}`); + + verifyExtractedInputs(source, manifest.inputs); + rejectSourceCargoConfiguration(source, join(work, 'cargo-home')); + const { compiler: compilerModule, settings } = await loadVerifiedBuildModules(manifest.inputs, runningInputFiles(), source); + if (canonicalJson(manifest.settings) !== canonicalJson(settings)) throw new Error('reader build settings differ from the closed implementation'); + compilerModule.validateCompilerManifest(manifest); compilerModule.validateCompilerLocator(locator); + compilerModule.validateWindowsPath(outputRoot, 'output root'); + const npmClosure = verifyClosure('npm-cache', manifest, locator, compilerModule.inventory); + const cargoSourceClosure = verifyClosure('cargo-home', manifest, locator, compilerModule.inventory); + const auditScript = runningInputFiles()['compiler-audit-script']; + const host = compilerModule.discoverSystemHost({ locator, manifest, auditScript, workRoot: work }); + const compiler = compilerModule.materializeCompiler({ manifest, locator, workRoot: work, host }); + const npmCache = materializeClosure('npm-cache', npmClosure, join(work, 'npm-cache'), manifest, compilerModule.inventory); + const cargoClosure = materializeClosure('cargo-home', cargoSourceClosure, join(work, 'cargo-closure'), manifest, compilerModule.inventory); + const cargoHome = join(work, 'cargo-home'); mkdirSync(cargoHome); + rejectSourceCargoConfiguration(source, cargoHome); + const cargoBuild = createCargoBuild({ compiler, workRoot: work, sourceRoot: source, cargoHome, cargoClosure, tempRoot }); + const { cargoVendor, env: controlled, args: cargoArgs } = cargoBuild; + for (const path of [source, cargoHome, cargoVendor, compiler.root]) { + if (!compilerModule.beneath(realpathSync.native(path), realpathSync.native(work))) { + throw new Error('native compiler input escaped the mapped work root'); + } + } + const rustArgs = controlled.CARGO_ENCODED_RUSTFLAGS.split('\x1f'); + const roots = [[source, ''], [work, ''], [output, ''], + [cargoHome, ''], [cargoVendor, ''], [cargoClosure, ''], + [compiler.root, '']]; + const logicalRustFlags = normalizeBuildText(rustArgs.join(' '), roots); + if (logicalRustFlags !== WINDOWS_LOGICAL_RUST_FLAGS) throw new Error('compiler flags differ from the closed Windows build contract'); + const logicalNativeFlags = normalizeBuildText(controlled.CL, [[work, '']]); + if (logicalNativeFlags !== WINDOWS_LOGICAL_NATIVE_FLAGS) throw new Error('native flags differ from the closed Windows build contract'); + const audits = []; + const executeCompiler = (id, args, label) => { + const result = compilerModule.runAuditedCompiler({ compiler, toolPath: compiler.tools[id], args, label, + cwd: source, env: controlled, auditScript, evidenceRoot: evidence, targetRoot: join(work, 'cargo-target') }); + audits.push({ label, path: portable(relative(output, result.evidencePath)), sha256: result.evidenceSha256 }); + return result.text; + }; + const cargoVersion = executeCompiler('cargo', ['--version'], 'cargo-version').trim(); + const rustVersion = executeCompiler('rustc', ['--version', '--verbose'], 'rust-version').trim(); + verifyRustHostVersion(rustVersion); + if (!/^cargo 1\.95\.0\b/.test(cargoVersion)) { + throw new Error(`pinned Rust toolchain version mismatch: ${cargoVersion}; ${rustVersion}`); + } + for (const query of ['sysroot', 'target-libdir']) { + const path = executeCompiler('rustc', ['--print', query], `rust-${query}`).trim(); + if (!existsSync(path) || !compilerModule.beneath(realpathSync.native(path), join(compiler.root, 'rust'))) throw new Error(`rustc ${query} escaped its private compiler`); + } + const adapter = compilerModule.prepareNativeArchiveAdapter(work); + executeCompiler('rustc', [adapter.source, '--crate-name', 'aware_native_archive_adapter', '--edition=2021', + '-C', 'opt-level=2', '-C', 'debuginfo=0', '-C', `linker=${compiler.tools.link}`, ...rustArgs, + '-o', adapter.executable], 'native-archive-adapter-build'); + const nativeArchiveAdapter = compilerModule.nativeArchiveAdapterRecord(output); + const npmEnv = { + SystemRoot: controlled.SystemRoot, WINDIR: controlled.WINDIR, ComSpec: controlled.ComSpec, + PATHEXT: controlled.PATHEXT, PATH: dirname(authority.tools.node), TEMP: tempRoot, TMP: tempRoot, + npm_config_cache: npmCache, npm_config_offline: 'true', npm_config_ignore_scripts: 'true', + npm_config_audit: 'false', npm_config_fund: 'false', npm_config_update_notifier: 'false', + npm_config_logs_dir: join(work, 'npm-logs'), npm_config_logs_max: '0', + }; + const readerRoot = join(source, 'cli-connection-reader'); + run(authority.tools.node, [authority.tools['npm-cli'], 'ci', '--offline', '--ignore-scripts'], { cwd: readerRoot, env: npmEnv }); + + rejectSourceCargoConfiguration(source, cargoHome); + const cargoLog = executeCompiler('cargo', cargoArgs, 'cargo-build'); + compilerModule.verifyNativeArchiveAdapter(nativeArchiveAdapter, + JSON.parse(readFileSync(join(evidence, 'cargo-build-audit.local.json'), 'utf8')), output); + compilerModule.verifyPrivateCompiler(compiler); + if (canonicalJson(compilerModule.inventory(cargoClosure)) !== canonicalJson(manifest.closures['cargo-home'].files)) throw new Error('private Cargo source changed during compilation'); + assertVerboseCargoProof(`${cargoArgs.join(' ')}\n${cargoLog}`, rustArgs, compiler.tools.rustc); + const normalizedCargo = normalizeBuildText(` ${cargoArgs.join(' ')}\n${cargoLog}`, roots); + writeFileSync(join(evidence, 'cargo-verbose.local.txt'), cargoLog, 'utf8'); + writeFileSync(join(evidence, 'cargo-verbose.normalized.txt'), normalizedCargo, 'utf8'); + + const readerLocator = join(work, 'reader-locator.local.json'); + writeFileSync(readerLocator, canonicalJson({ + schema: 'aware-windows-repro-locator/v1', tools: { + node: authority.tools.node, + postject: join(readerRoot, 'node_modules', 'postject', 'dist', 'cli.js'), + 'web-ifc-wasm': join(readerRoot, 'node_modules', 'web-ifc', 'web-ifc-node.wasm'), + }, + }), 'utf8'); + for (const [id, path] of Object.entries(JSON.parse(readFileSync(readerLocator, 'utf8')).tools)) { + const expected = manifest.tools[id]?.sha256; if (sha256File(path) !== expected) throw new Error(`installed reader tool differs: ${id}`); + } + run(authority.tools.node, [join(readerRoot, 'build-internal-repro.mjs'), + '--manifest', manifestPath, '--locator', readerLocator, '--output', join(artifacts, 'reader')], { cwd: readerRoot, env: { + SystemRoot: controlled.SystemRoot, WINDIR: controlled.WINDIR, ComSpec: controlled.ComSpec, + PATHEXT: controlled.PATHEXT, PATH: dirname(authority.tools.node), TEMP: tempRoot, TMP: tempRoot, + } }); + + const awareSource = join(work, 'cargo-target', 'release', 'aware.exe'); + if (!existsSync(awareSource)) throw new Error('Cargo did not produce aware.exe'); + copyFileSync(awareSource, join(artifacts, 'aware.exe')); + verifyConsumedClosure('npm-cache', npmCache, manifest, compilerModule.inventory); + const builderManifestRecord = writeBuilderManifestEvidence({ artifactsRoot: artifacts, manifestText }); + const receipt = { + schema: 'aware-windows-runtime-build-receipt/v1', + buildId: sha256Bytes(Buffer.from(manifestText)), builderManifestSha256: sha256Bytes(Buffer.from(manifestText)), + source: manifest.source, inputs: manifest.inputs, target: TARGET, + compiler: compilerModule.compilerSummary(manifest), + flags: { rust: logicalRustFlags, native: logicalNativeFlags, cargo: ['--release', '--locked', '--offline'] }, + outputs: { + 'aware.exe': { size: lstatSync(join(artifacts, 'aware.exe')).size, sha256: sha256File(join(artifacts, 'aware.exe')) }, + 'builder-manifest.json': builderManifestRecord, + 'reader/build-receipt.json': { + size: lstatSync(join(artifacts, 'reader', 'build-receipt.json')).size, + sha256: sha256File(join(artifacts, 'reader', 'build-receipt.json')), + }, + }, + commands: { + cargo: cargoBuild.command, + reader: ' /cli-connection-reader/build-internal-repro.mjs --manifest --locator --output /reader', + }, + unsignedTestMedia: true, + }; + writeFileSync(join(artifacts, 'build-receipt.json'), canonicalJson(receipt), 'utf8'); + writeFileSync(join(evidence, 'compiler-provenance.json'), canonicalJson({ schema: 'aware-compiler-provenance/v1', + source: manifest.source, buildId: receipt.buildId, compiler: receipt.compiler, audits, + artifacts: inventory(artifacts), nativeArchiveAdapter }), 'utf8'); + return receipt; +} + +if (realpathSync(scriptPath) === realpathSync(process.argv[1])) { + const args = parseArgs(process.argv.slice(2)); + await buildWindowsInternal({ manifestPath: resolve(args.manifest), locatorPath: resolve(args.locator), outputRoot: resolve(args.output) }); +} diff --git a/cli/build-windows-internal-repro.test.mjs b/cli/build-windows-internal-repro.test.mjs new file mode 100644 index 000000000..36d933ac2 --- /dev/null +++ b/cli/build-windows-internal-repro.test.mjs @@ -0,0 +1,242 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + assertVerboseCargoProof, cargoArguments, closedGitEnvironment, COMMAND_OUTPUT_BUFFER_BYTES, + controlledEnvironment, materializeClosure, rejectedAmbientKeys, SOURCE_PATHS, + writeBuilderManifestEvidence, rustCompilerArguments, verifiedVendorDirectory, normalizeBuildText, + verifyExtractedInputs, + createCargoBuild, verifyCargoVendorBinding, logicalCargoCommand, LOGICAL_CARGO_COMMAND, + runningInputFiles, loadVerifiedBuildModules, + nativeCompilerEnvironmentFlags, WINDOWS_LOGICAL_NATIVE_FLAGS, verifyRustHostVersion, rejectSourceCargoConfiguration, +} from './build-windows-internal-repro.mjs'; + +const fakeCompiler = () => ({ tools: { rustc: 'RUSTC', rustdoc: 'RUSTDOC', cl: 'CL', lib: 'LIB', link: 'LINK' }, + host: { windows: 'SYSTEM', system32: 'SYSTEM32' }, + environment: { PATH: 'PRIVATE_PATH', INCLUDE: 'PRIVATE_INCLUDE', LIB: 'PRIVATE_LIB', LIBPATH: 'PRIVATE_LIBPATH', PATHEXT: '.EXE', _NO_DEBUG_HEAP: '1' } }); + +test('verbose command evidence has an explicit bounded buffer large enough for a full Cargo proof', () => { + assert.equal(COMMAND_OUTPUT_BUFFER_BYTES, 128 * 1024 * 1024); +}); + +test('builder extracts only the two closed runtime source roots', () => { + assert.deepEqual(SOURCE_PATHS, ['cli', 'cli-connection-reader']); + assert.equal(Object.isFrozen(SOURCE_PATHS), true); +}); + +test('Cargo invocation is locked, offline, release, verbose, and Windows-specific', () => { + assert.deepEqual(cargoArguments('C:/src/cli/Cargo.toml', 'C:\\closure\\vendor'), [ + 'build', '--manifest-path', 'C:/src/cli/Cargo.toml', '--release', '--locked', '--offline', + '--config', 'source.crates-io.replace-with="vendored-sources"', + '--config', 'source.vendored-sources.directory="C:/closure/vendor"', + '--verbose', '--verbose', + ]); +}); + +test('controlled environment owns reproducible Rust and native MSVC flags', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-vendor-')); + mkdirSync(join(root, 'vendor')); + try { + const locator = { tools: { rustc: 'RUSTC', rustdoc: 'RUSTDOC', cl: 'CL', lib: 'LIB' }, environment: { + PATH: 'PATH', INCLUDE: 'INCLUDE', LIB: 'LIBS', LIBPATH: 'LIBPATH', SystemRoot: 'SYSTEM', + WINDIR: 'WINDOWS', ComSpec: 'CMD', PATHEXT: '.EXE', + } }; + const vendor = verifiedVendorDirectory(root); + const options = { compiler: fakeCompiler(), workRoot: 'C:\\WORK', sourceRoot: 'C:\\WORK\\SOURCE', cargoHome: 'C:\\WORK\\CARGO', cargoVendor: vendor, tempRoot: 'TEMP' }; + const env = controlledEnvironment(options); + assert.equal(env.RUSTFLAGS, undefined); + assert.deepEqual(env.CARGO_ENCODED_RUSTFLAGS.split('\x1f'), [ + '-C', 'link-arg=/Brepro', + '--remap-path-prefix=C:\\WORK=', '--remap-path-prefix=C:/WORK=', + '--remap-path-prefix=C:\\WORK\\SOURCE=', '--remap-path-prefix=C:/WORK/SOURCE=', + '--remap-path-prefix=C:\\WORK\\CARGO=', '--remap-path-prefix=C:/WORK/CARGO=', + ...[...new Set([vendor, vendor.replaceAll('\\', '/')])].map(path => `--remap-path-prefix=${path}=`), + ]); + for (const cargoVendor of [undefined, '', join(root, 'missing')]) { + assert.throws(() => controlledEnvironment({ ...options, cargoVendor }), /vendor path/); + } + writeFileSync(join(root, 'not-a-directory'), 'x'); + assert.throws(() => rustCompilerArguments({ ...options, cargoVendor: join(root, 'not-a-directory') }), /vendor path/); + assert.equal(env.CFLAGS, '/Brepro'); + assert.equal(normalizeBuildText(env.CL, [['C:\\WORK', '']]), WINDOWS_LOGICAL_NATIVE_FLAGS); + assert.equal(env.AR, join('C:\\WORK', 'cargo-target', 'native-tools', 'aware-lib.exe')); + assert.equal(env.AR_x86_64_pc_windows_msvc, env.AR); + assert.equal(env.CARGO_NET_OFFLINE, 'true'); assert.equal(env.RUSTC, 'RUSTC'); + assert.equal(env.CARGO_BUILD_JOBS, '1'); + assert.equal(env._NO_DEBUG_HEAP, '1'); + assert.equal(env.NODE_OPTIONS, undefined); assert.equal(env.GOOGLE_CLIENT_SECRET, undefined); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('verified closure and vendor operands cannot silently become an empty Cargo home', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-vendor-root-')); + try { + assert.throws(() => verifiedVendorDirectory(undefined), /closure is required/); + assert.throws(() => verifiedVendorDirectory(''), /closure is required/); + assert.throws(() => verifiedVendorDirectory(root), /existing directory/); + writeFileSync(join(root, 'vendor'), 'not a directory'); + assert.throws(() => verifiedVendorDirectory(root), /existing directory/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('the production Cargo contract rejects divergent existing vendor roots and missing config operands', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-cargo-contract-')); + try { + const closures = [join(root, 'sealed a'), join(root, 'sealed b')]; + for (const closure of closures) mkdirSync(join(closure, 'vendor'), { recursive: true }); + const locator = { tools: { rustc: 'RUSTC', rustdoc: 'RUSTDOC', cl: 'CL', lib: 'LIB' }, environment: { + PATH: 'PATH', INCLUDE: 'INCLUDE', LIB: 'LIBS', LIBPATH: 'LIBPATH', SystemRoot: 'SYSTEM', + WINDIR: 'WINDOWS', ComSpec: 'CMD', PATHEXT: '.EXE', + } }; + // This is a Windows command contract even when its pure tests run on Linux. + const options = { compiler: fakeCompiler(), workRoot: 'C:\\WORK', sourceRoot: join(root, 'work', 'source'), cargoHome: join(root, 'work', 'cargo-home'), tempRoot: join(root, 'temp') }; + const [a, b] = closures.map(cargoClosure => createCargoBuild({ ...options, cargoClosure })); + assert.equal(a.command, LOGICAL_CARGO_COMMAND); + assert.doesNotThrow(() => verifyCargoVendorBinding(a)); + const isVendor = arg => arg.startsWith('source.vendored-sources.directory='); + const divergentArgs = a.args.map(arg => isVendor(arg) ? b.args.find(isVendor) : arg); + assert.throws(() => verifyCargoVendorBinding({ ...a, args: divergentArgs }), /vendor operand differs/); + assert.throws(() => verifyCargoVendorBinding({ ...a, env: b.env }), /vendor remaps differ/); + for (let index = 0; index < a.args.length; index++) { + if (a.args[index] !== '--config') continue; + const missingOperand = [...a.args]; missingOperand.splice(index, 2); + assert.throws(() => logicalCargoCommand(missingOperand, [[options.sourceRoot, ''], [a.cargoVendor, '']]), /closed build contract/); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('logical evidence replaces specific roots first in both Windows spellings and case variants', () => { + const roots = [['C:\\WORK', ''], ['C:\\WORK\\source', ''], ['C:\\SEALED A\\vendor', '']]; + assert.equal(normalizeBuildText('C:\\WORK\\source\\main.rs c:/work/source/main.rs C:\\SEALED A\\vendor\\dep.rs', roots), + '\\main.rs /main.rs \\dep.rs'); + assert.equal(normalizeBuildText('C:\\\\SEALED A\\\\vendor\\\\dep.rs', roots), '\\\\dep.rs'); +}); + +test('a new source builder cannot be built using an old running script and lock-only manifest', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-builder-source-')); + try { + mkdirSync(join(root, 'cli')); mkdirSync(join(root, 'cli-connection-reader')); + const files = { 'aware-cargo-lock': join(root, 'cli', 'Cargo.lock'), 'reader-package-lock': join(root, 'cli-connection-reader', 'package-lock.json'), ...runningInputFiles(join(root, 'cli', 'build-windows-internal-repro.mjs')) }; + for (const [id, path] of Object.entries(files)) writeFileSync(path, id); + const running = join(root, 'running', 'cli', 'build-windows-internal-repro.mjs'); + mkdirSync(join(root, 'running', 'cli'), { recursive: true }); mkdirSync(join(root, 'running', 'cli-connection-reader')); + for (const [id, path] of Object.entries(runningInputFiles(running))) writeFileSync(path, id); + const inputs = Object.fromEntries(Object.entries(files).map(([id, path]) => [id, createHash('sha256').update(readFileSync(path)).digest('hex')])); + assert.doesNotThrow(() => verifyExtractedInputs(root, inputs, running)); + writeFileSync(files['builder-script'], 'new source builder'); + assert.throws(() => verifyExtractedInputs(root, inputs, running), /extracted source/); + inputs['builder-script'] = createHash('sha256').update(readFileSync(files['builder-script'])).digest('hex'); + assert.throws(() => verifyExtractedInputs(root, inputs, running), /extracted source/); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test('offline cache use is isolated in a verified private copy', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-builder-closure-')); + try { + const source = join(root, 'source'); const destination = join(root, 'destination'); + mkdirSync(join(source, 'cache'), { recursive: true }); + writeFileSync(join(source, 'cache', 'index'), 'closed'); + const digest = createHash('sha256').update('closed').digest('hex'); + const manifest = { closures: { cache: { files: [{ path: 'cache/index', size: 6, sha256: digest }] } } }; + assert.equal(materializeClosure('cache', source, destination, manifest), destination); + writeFileSync(join(destination, 'cache', 'index'), 'mutate'); + assert.equal(readFileSync(join(source, 'cache', 'index'), 'utf8'), 'closed'); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test('ambient authority detector covers compiler, npm, dotnet, and credentials', () => { + assert.deepEqual(rejectedAmbientKeys({ PATH: 'ok', LINK: 'poison', npm_config_cache: 'x', COREHOST_TRACE: '1' }), + ['COREHOST_TRACE', 'LINK', 'PATH', 'npm_config_cache']); +}); + +test('Git cannot consult host configuration, templates, prompts, or network transports', () => { + const env = closedGitEnvironment({ SystemRoot: 'WINDOWS' }); + assert.equal(env.GIT_CONFIG_NOSYSTEM, '1'); + assert.equal(env.GIT_CONFIG_GLOBAL, 'NUL'); + assert.equal(env.GIT_CONFIG_COUNT, '0'); + assert.equal(env.GIT_ALLOW_PROTOCOL, 'file'); + assert.equal(env.GIT_TERMINAL_PROMPT, '0'); +}); + +test('verbose proof goes red if the actual command loses /Brepro or a locked flag', () => { + const header = 'cargo build --release --locked --offline'; + const command = args => ' Running `set CARGO=private&& "C:\\private compiler\\rustc.exe" --crate-name probe -C link-arg=/Brepro '+args+'`'; + const complete = header+'\n'+command(''); + assert.doesNotThrow(() => assertVerboseCargoProof(complete)); + assert.throws(() => assertVerboseCargoProof(complete.replace('/Brepro', '/DEBUG')), /omitted \/Brepro/); + assert.throws(() => assertVerboseCargoProof(complete.replace('--locked', '')), /locked mode/); + const vendorArgs = ['--remap-path-prefix=C:\\sealed cache\\vendor=', '--remap-path-prefix=C:/sealed cache/vendor=']; + const mapped = header+'\n'+command(vendorArgs.map(a=>'"'+a+'"').join(' ')); + assert.doesNotThrow(() => assertVerboseCargoProof(mapped, vendorArgs, 'C:\\private compiler\\rustc.exe')); + const rawExecutable = mapped.replace('"C:\\private compiler\\rustc.exe"', 'C:\\private compiler\\rustc.exe'); + assert.doesNotThrow(() => assertVerboseCargoProof(rawExecutable, vendorArgs, 'C:\\private compiler\\rustc.exe'), 'real Windows Cargo display leaves the spaced executable unquoted'); + const buildScript = ' Running `set RUSTC=C:\\private compiler\\rustc.exe&& C:\\target path\\build-script-build.exe`'; + assert.doesNotThrow(() => assertVerboseCargoProof(rawExecutable+'\n'+buildScript, vendorArgs, 'C:\\private compiler\\rustc.exe'), 'an environment assignment does not turn a build script into rustc'); + for (const missing of vendorArgs) { + assert.throws(() => assertVerboseCargoProof(header+'\n'+command(vendorArgs.filter(arg => arg !== missing).join(' ')), vendorArgs), /compiler remap/); + } + assert.throws(() => assertVerboseCargoProof(mapped+'\n'+command(''), vendorArgs), /compiler remap/, 'a fully mapped target cannot cover a host compilation'); + assert.throws(() => assertVerboseCargoProof(mapped+'\n'+command('').slice(0, -1), vendorArgs), /unparseable rustc/, 'a malformed host command cannot disappear'); + assert.throws(() => assertVerboseCargoProof(mapped+'\n'+command('"unterminated'), vendorArgs), /unparseable rustc/); + assert.throws(() => assertVerboseCargoProof(mapped.replaceAll('=', '=-wrong'), vendorArgs), /compiler remap/, 'a partial remap match is not proof'); + assert.throws(() => assertVerboseCargoProof(mapped.replace('link-arg=/Brepro', 'link-arg=/Brepro-wrong'), vendorArgs), /omitted \/Brepro/); + assert.throws(() => assertVerboseCargoProof(header+' '+vendorArgs.join(' ')), /actual rustc compilations/); + assert.throws(() => assertVerboseCargoProof(mapped, vendorArgs, 'C:\\different\\rustc.exe'), /different rustc/); +}); + +test('native mappings fit CL limits and quote supported long Unicode paths', () => { + const work = 'C:\\'+('long Łódź directory '.repeat(9)).trim(); + const flags = nativeCompilerEnvironmentFlags(work); + assert.ok(flags.length <= 1024); + assert.equal(normalizeBuildText(flags, [[work, '']]), WINDOWS_LOGICAL_NATIVE_FLAGS); + for (const bad of ['C:\\root"injection', 'C:\\root;extra', '\\\\server\\share', 'C:relative', 'C:\\'+('a'.repeat(200))]) { + assert.throws(() => nativeCompilerEnvironmentFlags(bad), /unsafe bootstrap/); + } +}); + +test('implicit native Cargo target requires the exact audited Rust host', () => { + const proof = 'rustc 1.95.0 (59807616e 2026-04-14)\nbinary: rustc\nhost: x86_64-pc-windows-msvc\nrelease: 1.95.0\n'; + assert.doesNotThrow(() => verifyRustHostVersion(proof)); + for (const bad of [proof.replace('1.95.0', '1.94.0'), proof.replace('x86_64-pc-windows-msvc', 'aarch64-pc-windows-msvc'), proof+'host: x86_64-pc-windows-msvc\n', 'rustc 1.95.0']) { + assert.throws(() => verifyRustHostVersion(bad), /version\/host mismatch/); + } +}); + +test('Cargo source, private home and every ancestor config refuse before a launch', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-config-refusal-')); + const source = join(root, 'output', 'work', 'source'), home = join(root, 'output', 'work', 'cargo-home'); + mkdirSync(source, {recursive:true}); mkdirSync(home); + try { + assert.doesNotThrow(() => rejectSourceCargoConfiguration(source, home)); + for (const directory of [join(root,'.cargo'), join(source,'.cargo'), join(source,'cli','.cargo'), home]) { + mkdirSync(directory,{recursive:true}); + for (const name of ['config','config.toml']) { + const file=join(directory,name); writeFileSync(file,'[build]\ntarget="x86_64-pc-windows-msvc"\n'); + let launched=false; + assert.throws(()=>{rejectSourceCargoConfiguration(source,home);launched=true;},/Cargo configuration/); + assert.equal(launched,false); rmSync(file); + } + } + } finally { rmSync(root,{recursive:true,force:true}); } +}); + +test('builder manifest is retained byte-for-byte as independently digestible evidence', () => { + const root = mkdtempSync(join(tmpdir(), 'aware-builder-manifest-')); + try { + const manifestText = '{\n "schema": "aware-windows-repro-builder/v1"\n}\n'; + const record = writeBuilderManifestEvidence({ artifactsRoot: root, manifestText }); + assert.equal(readFileSync(join(root, 'builder-manifest.json'), 'utf8'), manifestText); + assert.equal(record.size, Buffer.byteLength(manifestText)); + assert.match(record.sha256, /^[0-9a-f]{64}$/); + assert.throws(() => writeBuilderManifestEvidence({ artifactsRoot: root, manifestText }), /EEXIST/); + assert.throws(() => writeBuilderManifestEvidence({ artifactsRoot: root, manifestText: '{"schema":"x"}\n' }), + /canonical JSON/); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/cli/create-windows-internal-repro-inputs.mjs b/cli/create-windows-internal-repro-inputs.mjs new file mode 100644 index 000000000..0c2da4ba8 --- /dev/null +++ b/cli/create-windows-internal-repro-inputs.mjs @@ -0,0 +1,104 @@ +#!/usr/bin/env node +// Turns local tool and closure paths into a path-free reproducible-builder manifest plus a +// local-only locator. The manifest digest is the immutable identity of the Windows build authority. +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, lstatSync, readFileSync, realpathSync, writeFileSync } from 'node:fs'; +import { isAbsolute, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { READER_BUILD_SETTINGS } from '../cli-connection-reader/repro-settings.mjs'; +import { CLOSURE_IDS, COMPILER_DESCRIPTOR, INPUT_IDS, NONCOMPILER_TOOL_IDS, exactKeys, + inventory as compilerInventory, validateCompilerManifest, validateCompilerLocator, validateWindowsPath } from './windows-compiler-closure.mjs'; + +const SCRIPT = fileURLToPath(import.meta.url); +const SHA1 = /^[0-9a-f]{40}$/; +const TOOL_IDS = NONCOMPILER_TOOL_IDS; +const sha256 = (path) => createHash('sha256').update(readFileSync(path)).digest('hex'); +const canonical = (value) => Array.isArray(value) ? value.map(canonical) + : value && typeof value === 'object' + ? Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])) : value; +export const canonicalJson = (value) => `${JSON.stringify(canonical(value), null, 2)}\n`; + +function requireFile(path, label) { + if (typeof path !== 'string' || !isAbsolute(path) || !existsSync(path) || !lstatSync(path).isFile()) { + throw new Error(`${label} must be an absolute existing file`); + } + return resolve(path); +} + +function requireDirectory(path, label) { + if (typeof path !== 'string' || !isAbsolute(path) || !existsSync(path) || !lstatSync(path).isDirectory()) { + throw new Error(`${label} must be an absolute existing directory: ${path}`); + } + return resolve(path); +} + +export const inventory = compilerInventory; + +export function createWindowsBuilderRecords(input) { + if (input?.schema !== 'aware-windows-repro-builder-inputs/v1') throw new Error('unsupported builder-input schema'); + exactKeys(input, ['schema', 'source', 'inputs', 'tools', 'closures'], 'builder inputs'); + exactKeys(input.source, ['commit', 'tree', 'bundle'], 'builder input source'); + exactKeys(input.inputs, INPUT_IDS, 'builder script/lock inputs'); + exactKeys(input.tools, TOOL_IDS, 'builder noncompiler tools'); + exactKeys(input.closures, CLOSURE_IDS, 'builder compiler/dependency roots'); + // Physical Windows admission occurs before stat/hash calls. POSIX fixtures exercise + // only portable manifest semantics; production builders require Windows. + if (process.platform === 'win32') { + for (const [id, path] of [...Object.entries(input.inputs), ...Object.entries(input.tools), ...Object.entries(input.closures), ['bundle', input.source.bundle]]) validateWindowsPath(path, id); + } + if (!SHA1.test(input.source?.commit ?? '') || !SHA1.test(input.source?.tree ?? '')) throw new Error('invalid source commit/tree'); + const sourceBundle = requireFile(input.source.bundle, 'source bundle'); + const locks = Object.fromEntries(INPUT_IDS.map(id => [id, requireFile(input.inputs[id], id)])); + const tools = Object.fromEntries(TOOL_IDS.map((id) => [id, requireFile(input.tools?.[id], `${id} tool`)])); + const closures = Object.fromEntries(CLOSURE_IDS.map(id => [id, requireDirectory(input.closures[id], id)])); + const manifest = { + schema: 'aware-windows-repro-builder/v1', platform: 'win32', arch: 'x64', + nodeVersion: '24.14.0', rustVersion: '1.95.0', target: 'x86_64-pc-windows-msvc', + source: { commit: input.source.commit, tree: input.source.tree, bundleSha256: sha256(sourceBundle) }, + settings: READER_BUILD_SETTINGS, + compiler: COMPILER_DESCRIPTOR, + inputs: Object.fromEntries(Object.entries(locks).map(([id, path]) => [id, sha256(path)])), + tools: Object.fromEntries(TOOL_IDS.map((id) => [id, { id, sha256: sha256(tools[id]) }])), + closures: Object.fromEntries(Object.entries(closures).map(([id, root]) => [id, { files: compilerInventory(root) }])), + }; + validateCompilerManifest(manifest); + const manifestText = canonicalJson(manifest); + const buildId = createHash('sha256').update(manifestText).digest('hex'); + const locator = { + schema: 'aware-windows-repro-locator/v1', sourceBundle, tools, closures, + }; + if (process.platform === 'win32') validateCompilerLocator(locator); + return { manifest, manifestText, locator, locatorText: canonicalJson(locator), buildId }; +} + +function parseArgs(argv) { + const out = {}; + for (let index = 0; index < argv.length; index += 2) { + if (!argv[index]?.startsWith('--') || argv[index + 1] == null) { + throw new Error('arguments are --input FILE --manifest FILE --locator FILE'); + } + out[argv[index].slice(2)] = argv[index + 1]; + } + for (const key of ['input', 'manifest', 'locator']) if (!isAbsolute(out[key] ?? '')) throw new Error(`--${key} must be absolute`); + return out; +} + +function git(path, args) { + const result = spawnSync(path, args, { encoding: 'utf8', windowsHide: true }); + if (result.error || result.status !== 0) throw new Error(`git ${args.join(' ')} failed: ${result.error?.message ?? result.stderr}`); + return String(result.stdout); +} + +if (realpathSync.native(SCRIPT).toLowerCase() === realpathSync.native(process.argv[1]).toLowerCase()) { + const args = parseArgs(process.argv.slice(2)); + const input = JSON.parse(readFileSync(args.input, 'utf8')); + const records = createWindowsBuilderRecords(input); + const heads = git(records.locator.tools.git, ['bundle', 'list-heads', records.locator.sourceBundle]); + if (!heads.split(/\r?\n/).some((line) => line.startsWith(`${records.manifest.source.commit} `))) { + throw new Error('source bundle does not advertise the selected commit'); + } + writeFileSync(args.manifest, records.manifestText, { encoding: 'utf8', flag: 'wx' }); + writeFileSync(args.locator, records.locatorText, { encoding: 'utf8', flag: 'wx' }); + console.log(`AWARE Windows builder manifest ${records.buildId}`); +} diff --git a/cli/create-windows-internal-repro-inputs.test.mjs b/cli/create-windows-internal-repro-inputs.test.mjs new file mode 100644 index 000000000..e8f278f34 --- /dev/null +++ b/cli/create-windows-internal-repro-inputs.test.mjs @@ -0,0 +1,37 @@ +import assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import test from 'node:test'; +import { READER_BUILD_SETTINGS } from '../cli-connection-reader/repro-settings.mjs'; +import { createWindowsBuilderRecords } from './create-windows-internal-repro-inputs.mjs'; +import { compilerFixture } from './windows-compiler-fixture.mjs'; + +function fixture() { + const result = compilerFixture(); + writeFileSync(join(result.input.closures['npm-cache'], 'z'), 'z'); + writeFileSync(join(result.input.closures['npm-cache'], 'a'), 'a'); + return result; +} + +test('builder input compiler keeps local paths out of the canonical manifest', () => { + const f = fixture(); + try { + const first = createWindowsBuilderRecords(f.input); + const second = createWindowsBuilderRecords(f.input); + assert.equal(first.manifestText, second.manifestText); + assert.equal(first.buildId, second.buildId); + assert.equal(first.manifest.closures['npm-cache'].files.map((record) => record.path).join(','), 'a,cache,z'); + assert.deepEqual(first.manifest.settings, READER_BUILD_SETTINGS); + assert.ok(!first.manifestText.includes(f.root)); + assert.equal(JSON.parse(first.locatorText).sourceBundle, f.input.source.bundle); + } finally { rmSync(f.root, { recursive: true, force: true }); } +}); + +test('builder input compiler rejects symlinks inside offline closures', { skip: process.platform === 'win32' }, () => { + const f = fixture(); + try { + symlinkSync(join(f.root, 'source.bundle'), join(f.input.closures['npm-cache'], 'link')); + assert.throws(() => createWindowsBuilderRecords(f.input), /path-redirection/); + } finally { rmSync(f.root, { recursive: true, force: true }); } +}); diff --git a/cli/src/commands/app.rs b/cli/src/commands/app.rs index 8fc118f32..4ba6ae737 100644 --- a/cli/src/commands/app.rs +++ b/cli/src/commands/app.rs @@ -492,7 +492,8 @@ async fn run( }; } - // One-shot path. + // One-shot path. The reader fence was acquired above for both one-shot and long-running + // graphs so provider cleanup remains serialized across the complete run lifecycle. let log_path = log_path_for(&ctx.paths.logs_dir(), app_id, &instance, &run_id); let provenance = ProvenanceWriter::open(&log_path).await?; let artifact_dir = crate::runtime::provenance::artifact_dir_for( diff --git a/cli/windows-compiler-audit.ps1 b/cli/windows-compiler-audit.ps1 new file mode 100644 index 000000000..a83f3ccc0 --- /dev/null +++ b/cli/windows-compiler-audit.ps1 @@ -0,0 +1,289 @@ +param([Parameter(Mandatory=$true)][string]$RequestPath) +$ErrorActionPreference = 'Stop' +$request = Get-Content -LiteralPath $RequestPath -Raw -Encoding UTF8 | ConvertFrom-Json +if ([IntPtr]::Size -ne 8) { throw 'Compiler auditor requires Windows x64' } +# CodeDOM's legacy compiler cannot use all Unicode temporary paths. This is +# solely the authenticated bootstrap's scratch area; compiler environments below +# still come from the declared request, including their original TEMP/TMP paths. +$auditTempParent = [IO.Path]::GetFullPath([IO.Path]::Combine($env:SystemRoot, 'Temp')) +if ($env:TEMP -ne $auditTempParent -or $env:TMP -ne $auditTempParent -or $auditTempParent -match '[^\x20-\x7e]') { throw 'Invalid auditor temporary parent' } +if (([IO.File]::GetAttributes($auditTempParent) -band [IO.FileAttributes]::ReparsePoint) -ne 0) { throw 'Auditor temporary parent redirects elsewhere' } +$auditOwner = [Security.Principal.WindowsIdentity]::GetCurrent().User +$auditAcl = [Security.AccessControl.DirectorySecurity]::new() +$auditAcl.SetAccessRuleProtection($true, $false) +$auditAcl.SetOwner($auditOwner) +foreach ($identity in @($auditOwner, [Security.Principal.SecurityIdentifier]::new('S-1-5-18'))) { + $auditAcl.AddAccessRule([Security.AccessControl.FileSystemAccessRule]::new($identity, 'FullControl', 'ContainerInherit, ObjectInherit', 'None', 'Allow')) +} +$auditTempPath = [IO.Path]::Combine($auditTempParent, 'aware-compiler-audit-' + [Guid]::NewGuid().ToString('N')) +if ([IO.Directory]::Exists($auditTempPath) -or [IO.File]::Exists($auditTempPath)) { throw 'Auditor temporary path already exists' } +$auditDirectory = [IO.Directory]::CreateDirectory($auditTempPath, $auditAcl) +try { + if (($auditDirectory.Attributes -band [IO.FileAttributes]::ReparsePoint) -ne 0) { throw 'Auditor temporary directory redirects elsewhere' } + $actualAcl = $auditDirectory.GetAccessControl() + if (!$actualAcl.AreAccessRulesProtected -or $actualAcl.GetSecurityDescriptorSddlForm('Access, Owner') -ne $auditAcl.GetSecurityDescriptorSddlForm('Access, Owner')) { throw 'Auditor temporary access rules differ' } + $env:TEMP = $auditTempPath + $env:TMP = $auditTempPath +Add-Type -TypeDefinition @' +using System; +using System.IO; +using System.Text; +using System.Diagnostics; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using Microsoft.Win32.SafeHandles; + +public static class AwareCompilerAudit { + [StructLayout(LayoutKind.Sequential, CharSet=CharSet.Unicode)] struct Startup { + public uint cb; public string reserved, desktop, title; + public uint x,y,xSize,ySize,xChars,yChars,fill,flags; public ushort show, reservedSize; + public IntPtr reservedBytes, input, output, error; + } + [StructLayout(LayoutKind.Sequential)] struct ProcessInfo { public IntPtr process, thread; public uint pid, tid; } + [StructLayout(LayoutKind.Sequential)] struct Accounting { + public long user,kernel,periodUser,periodKernel; public uint faults,total,active,terminated; + } + public class Image { public uint pid,instance; public string path,kind,sha256; public long size,@event; } + public class ProcessRecord { public uint pid,instance; public long startEvent; public long? exitEvent; public string path,action="observed"; public uint? exitCode; } + public class DeniedImage { public string closure,relativePath,path,sha256; public long size; public uint exitCode; } + public class StartupPolicy { public string identity; public DeniedImage deniedImage; } + public class Report { + public string schema="aware-compiler-debug-audit/v3", error; + public long eventCount; + public StartupPolicy startupPolicy; + public bool complete; public uint exitCode=uint.MaxValue,totalProcesses,activeProcesses; + public List processes=new List(); + public List images=new List(); + } + // Windows may recycle a PID after EXIT. Only the active lifetime owns its + // debugger handle and initial breakpoint; history is identified by instance. + public class ProcessLifetimes { + class Active { public ProcessRecord record; public IntPtr handle; public bool breakpoint; } + readonly Report report; + readonly Dictionary active=new Dictionary(); + public ProcessLifetimes(Report report) { this.report=report; } + public int Count { get { return active.Count; } } + public bool RootExited { get { return report.processes.Count>0&&report.processes[0].exitCode.HasValue; } } + public void RequireNew(uint pid) { if(pid==0||active.ContainsKey(pid))throw new Exception("Duplicate active compiler process"); } + Active Current(uint pid) { Active value; if(!active.TryGetValue(pid,out value))throw new Exception("Event has no active compiler lifetime"); return value; } + public ProcessRecord RequireActive(uint pid) { return Current(pid).record; } + public IntPtr Handle(uint pid) { return Current(pid).handle; } + public ProcessRecord Begin(Image image,IntPtr handle) { + RequireNew(image.pid); + if(image.kind!="process")throw new Exception("Lifetime requires a process image"); + var record=new ProcessRecord {pid=image.pid,instance=(uint)report.processes.Count+1,path=image.path,startEvent=report.eventCount}; + image.instance=record.instance;image.@event=report.eventCount; + active.Add(record.pid,new Active {record=record,handle=handle}); + report.processes.Add(record);report.images.Add(image);return record; + } + public void Dll(Image image) { + var value=Current(image.pid); + if(image.kind!="dll")throw new Exception("Expected a DLL image"); + image.instance=value.record.instance;image.@event=report.eventCount;report.images.Add(image); + } + public ProcessRecord End(uint pid,uint exit) { + var record=Current(pid).record;record.exitCode=exit;record.exitEvent=report.eventCount; + active.Remove(pid); + if(record.instance==1)report.exitCode=exit; + // ContinueDebugEvent(EXIT) closes the debugger-provided process/thread + // handles. Do not close them here (Microsoft debugging-event contract). + return record; + } + public bool InitialBreakpoint(uint pid) { + var value=Current(pid);if(value.breakpoint)return false;value.breakpoint=true;return true; + } + } + [DllImport("kernel32.dll",CharSet=CharSet.Unicode)] static extern uint GetSystemWindowsDirectoryW(StringBuilder text,uint size); + [DllImport("kernel32.dll",CharSet=CharSet.Unicode)] static extern uint GetSystemDirectoryW(StringBuilder text,uint size); + [DllImport("kernel32.dll",SetLastError=true,CharSet=CharSet.Unicode)] static extern bool CreateProcessW(string app,StringBuilder command,IntPtr pa,IntPtr ta,bool inherit,uint flags,IntPtr environment,string cwd,ref Startup startup,out ProcessInfo process); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool WaitForDebugEventEx(IntPtr debugEvent,uint milliseconds); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool ContinueDebugEvent(uint pid,uint tid,uint status); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool DebugSetProcessKillOnExit(bool kill); + [DllImport("kernel32.dll",SetLastError=true,CharSet=CharSet.Unicode)] static extern uint GetFinalPathNameByHandleW(IntPtr file,StringBuilder path,uint size,uint flags); + [DllImport("kernel32.dll",SetLastError=true,CharSet=CharSet.Unicode)] static extern bool QueryFullProcessImageNameW(IntPtr process,uint flags,StringBuilder path,ref uint size); + [DllImport("psapi.dll",SetLastError=true,CharSet=CharSet.Unicode)] static extern uint GetMappedFileNameW(IntPtr process,IntPtr address,StringBuilder path,uint size); + [DllImport("kernel32.dll",SetLastError=true,CharSet=CharSet.Unicode)] static extern uint QueryDosDeviceW(string device,StringBuilder target,uint size); + [DllImport("kernel32.dll",SetLastError=true,CharSet=CharSet.Unicode)] static extern IntPtr CreateJobObjectW(IntPtr attributes,string name); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool SetInformationJobObject(IntPtr job,int kind,IntPtr information,uint size); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool AssignProcessToJobObject(IntPtr job,IntPtr process); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool QueryInformationJobObject(IntPtr job,int kind,out Accounting info,uint size,IntPtr returned); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool TerminateJobObject(IntPtr job,uint code); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool TerminateProcess(IntPtr process,uint code); + [DllImport("kernel32.dll")] static extern IntPtr GetStdHandle(int id); + [DllImport("kernel32.dll")] static extern IntPtr GetCurrentProcess(); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool DuplicateHandle(IntPtr sourceProcess,IntPtr source,IntPtr targetProcess,out IntPtr target,uint access,bool inherit,uint options); + [DllImport("kernel32.dll",SetLastError=true)] static extern bool CloseHandle(IntPtr handle); + static void Check(bool ok,string operation) { if(!ok) throw new System.ComponentModel.Win32Exception(Marshal.GetLastWin32Error(),operation); } + public static string[] Host() { + var windows=new StringBuilder(32768); var system=new StringBuilder(32768); + uint a=GetSystemWindowsDirectoryW(windows,32768),b=GetSystemDirectoryW(system,32768); + if(a==0||a>=32768||b==0||b>=32768) throw new Exception("Windows directory API failed"); + return new string[]{Path.GetFullPath(windows.ToString()),Path.GetFullPath(system.ToString())}; + } + static string DosPath(string path) { + if(path.StartsWith(@"\\?\",StringComparison.Ordinal)) path=path.Substring(4); + if(path.StartsWith(@"\Device\",StringComparison.OrdinalIgnoreCase)) { + foreach(string drive in Environment.GetLogicalDrives()) { + var target=new StringBuilder(32768); + if(QueryDosDeviceW(drive.Substring(0,2),target,32768)>0) { + string prefix=target.ToString(); + if(path.StartsWith(prefix+"\\",StringComparison.OrdinalIgnoreCase)) return Path.GetFullPath(drive.Substring(0,2)+path.Substring(prefix.Length)); + } + } + throw new Exception("Unresolved mapped-file device path"); + } + if(path.Length<3||path[1]!=':'||path[2]!='\\') throw new Exception("Image lacks a local canonical path"); + return Path.GetFullPath(path); + } + static Image Capture(uint pid,string kind,IntPtr file,IntPtr process,IntPtr address) { + var text=new StringBuilder(32768); uint length=0; + if(file!=IntPtr.Zero&&file!=new IntPtr(-1)) length=GetFinalPathNameByHandleW(file,text,32768,0); + if(length==0) { + text.Clear(); + if(kind=="process") { length=32768; Check(QueryFullProcessImageNameW(process,0,text,ref length),"QueryFullProcessImageName"); } + else length=GetMappedFileNameW(process,address,text,32768); + } + if(length==0||length>=32768) throw new Exception("Unresolved compiler image provenance"); + string path=DosPath(text.ToString()); + using(var stream=file!=IntPtr.Zero&&file!=new IntPtr(-1) + ? new FileStream(new SafeFileHandle(file,false),FileAccess.Read) + : new FileStream(path,FileMode.Open,FileAccess.Read,FileShare.ReadWrite|FileShare.Delete)) + using(var sha=SHA256.Create()) { + stream.Position=0; + return new Image {pid=pid,kind=kind,path=path,size=stream.Length,sha256=BitConverter.ToString(sha.ComputeHash(stream)).Replace("-","").ToLowerInvariant()}; + } + } + public static string Quote(string value) { + var result=new StringBuilder("\""); int slashes=0; + foreach(char c in value) { + if(c=='\\') {slashes++;continue;} + if(c=='\"') {result.Append('\\',slashes*2+1);result.Append(c);slashes=0;continue;} + result.Append('\\',slashes);slashes=0;result.Append(c); + } + result.Append('\\',slashes*2);result.Append('"');return result.ToString(); + } + static IntPtr Inherit(int id) { + IntPtr result; var current=GetCurrentProcess(); + Check(DuplicateHandle(current,GetStdHandle(id),current,out result,0,true,2),"Duplicate standard handle"); return result; + } + public static Report Run(string executable,string[] args,string cwd,string environment,int timeoutMs,StartupPolicy policy) { + var report=new Report(); var lifetimes=new ProcessLifetimes(report); + IntPtr job=IntPtr.Zero,limits=IntPtr.Zero,eventBuffer=IntPtr.Zero,env=IntPtr.Zero; + IntPtr input=IntPtr.Zero,output=IntPtr.Zero,error=IntPtr.Zero; ProcessInfo initial=new ProcessInfo(); + bool assigned=false; var clock=Stopwatch.StartNew(); + try { + if(IntPtr.Size!=8) throw new Exception("Debugger requires Windows x64"); + if(policy==null||policy.identity!="aware-private-msvc-telemetry-denial/v1") throw new Exception("Unknown compiler startup policy"); + DeniedImage denied=policy.deniedImage; + if(denied!=null) { + if(denied.closure!="compiler-msvc-bin"||!String.Equals(denied.relativePath,"vctip.exe",StringComparison.OrdinalIgnoreCase) + ||denied.exitCode!=0xe0000488u||denied.size<=0||!System.Text.RegularExpressions.Regex.IsMatch(denied.sha256??"","^[0-9a-f]{64}$") + ||!String.Equals(Path.GetFileName(denied.path),"vctip.exe",StringComparison.OrdinalIgnoreCase) + ||DosPath(denied.path)!=denied.path) throw new Exception("Invalid private telemetry policy"); + if(String.Equals(DosPath(executable),denied.path,StringComparison.OrdinalIgnoreCase)) throw new Exception("Root cannot be denied telemetry"); + } + // These are the effective values used below, including an explicit null if absent. + report.startupPolicy=policy; + job=CreateJobObjectW(IntPtr.Zero,null);Check(job!=IntPtr.Zero,"Create owned job"); + // x64 JOBOBJECT_EXTENDED_LIMIT_INFORMATION: basic flags at offset 16; total size 144. + limits=Marshal.AllocHGlobal(144); for(int i=0;i<144;i++)Marshal.WriteByte(limits,i,0); + Marshal.WriteInt32(limits,16,0x2000); // KILL_ON_JOB_CLOSE; no breakaway flags. + Check(SetInformationJobObject(job,9,limits,144),"Set owned job limits"); + input=Inherit(-10);output=Inherit(-11);error=Inherit(-12); + var startup=new Startup {cb=(uint)Marshal.SizeOf(typeof(Startup)),flags=0x100,input=input,output=output,error=error}; + var command=new StringBuilder(Quote(executable)); foreach(string arg in args)command.Append(" ").Append(Quote(arg)); + env=Marshal.StringToHGlobalUni(environment); eventBuffer=Marshal.AllocHGlobal(176); + Check(CreateProcessW(executable,command,IntPtr.Zero,IntPtr.Zero,true,0x08000401,env,cwd,ref startup,out initial),"Create private debug process"); + Check(AssignProcessToJobObject(job,initial.process),"Assign private debug process to job");assigned=true; + Check(DebugSetProcessKillOnExit(true),"Keep private debugger kill-on-exit"); + while(true) { + if(clock.ElapsedMilliseconds>timeoutMs)throw new Exception("Compiler audit deadline exceeded"); + if(!WaitForDebugEventEx(eventBuffer,100)) { + int code=Marshal.GetLastWin32Error(); if(code!=121)throw new System.ComponentModel.Win32Exception(code,"WaitForDebugEventEx"); + if(lifetimes.RootExited&&lifetimes.Count==0)break; continue; + } + uint kind=(uint)Marshal.ReadInt32(eventBuffer,0),pid=(uint)Marshal.ReadInt32(eventBuffer,4),tid=(uint)Marshal.ReadInt32(eventBuffer,8); + report.eventCount++; + if(kind==3) { + lifetimes.RequireNew(pid); + if(report.processes.Count==0&&pid!=initial.pid)throw new Exception("First creation is not the requested root"); + } else lifetimes.RequireActive(pid); + uint status=0x00010002; // DBG_CONTINUE for nonexception events. + if(kind==3) { + IntPtr file=Marshal.ReadIntPtr(eventBuffer,16),process=Marshal.ReadIntPtr(eventBuffer,24); + try { + Image image=Capture(pid,"process",file,process,Marshal.ReadIntPtr(eventBuffer,40)); + var record=lifetimes.Begin(image,process); + if(denied!=null&&String.Equals(image.path,denied.path,StringComparison.OrdinalIgnoreCase)) { + if(record.instance==1||image.size!=denied.size||image.sha256!=denied.sha256) throw new Exception("Telemetry creation differs from private authority"); + // The creation event has not been continued: no user entry point has run. + Check(TerminateProcess(process,denied.exitCode),"Deny private telemetry startup"); + record.action="blocked-telemetry"; + } + } finally {if(file!=IntPtr.Zero&&file!=new IntPtr(-1))CloseHandle(file);} + } else if(kind==6) { + IntPtr file=Marshal.ReadIntPtr(eventBuffer,16); + try {lifetimes.Dll(Capture(pid,"dll",file,lifetimes.Handle(pid),Marshal.ReadIntPtr(eventBuffer,24)));} + finally {if(file!=IntPtr.Zero&&file!=new IntPtr(-1))CloseHandle(file);} + } else if(kind==5) { + lifetimes.End(pid,(uint)Marshal.ReadInt32(eventBuffer,16)); + } else if(kind==1) { + uint exception=(uint)Marshal.ReadInt32(eventBuffer,16); + status=exception==0x80000003 && lifetimes.InitialBreakpoint(pid)?0x00010002u:0x80010001u; + } else if(kind==9) throw new Exception("Compiler debugger reported a RIP event"); + else if(kind!=2&&kind!=4&&kind!=7&&kind!=8)throw new Exception("Unknown compiler debugger event"); + Check(ContinueDebugEvent(pid,tid,status),"ContinueDebugEvent"); + if(lifetimes.RootExited&&lifetimes.Count==0)break; + } + Accounting accounting;long exitDeadline=clock.ElapsedMilliseconds+2000; + do { + Check(QueryInformationJobObject(job,1,out accounting,(uint)Marshal.SizeOf(typeof(Accounting)),IntPtr.Zero),"Read owned job accounting"); + if(accounting.active==0||accounting.total!=(uint)report.processes.Count)break; + System.Threading.Thread.Sleep(10); + } while(clock.ElapsedMilliseconds Array.isArray(value) ? value.map(canonical) : value && typeof value === 'object' + ? Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : value; +export const canonicalJson = value => `${JSON.stringify(canonical(value), null, 2)}\n`; +export const digest = bytes => createHash('sha256').update(bytes).digest('hex'); +export const fileDigest = path => digest(readFileSync(path)); +// Embedded in authenticated helper bytes; never loaded from an ambient source or executable. +export const NATIVE_ARCHIVE_ADAPTER_SOURCE = String.raw` +use std::{env, fs, path::{Component, Path, PathBuf}, process::{Command, ExitCode}}; +use std::os::windows::fs::MetadataExt; + +fn ordinary(path: &Path) -> Result { + let text = path.to_str().ok_or("non-Unicode path")?; + Ok(PathBuf::from(text.strip_prefix(r"\\?\").unwrap_or(text))) +} +fn clean_components(path: &Path) -> Result<(), String> { + let mut current = PathBuf::new(); + for component in path.components() { + current.push(component); + if matches!(component, Component::Normal(_)) { + let meta = fs::symlink_metadata(¤t).map_err(|e| e.to_string())?; + if meta.file_attributes() & 0x400 != 0 { return Err("reparse path is forbidden".into()); } + } + } + Ok(()) +} +fn existing(path: &Path) -> Result { + clean_components(path)?; + ordinary(&fs::canonicalize(path).map_err(|e| e.to_string())?) +} +fn beneath(path: &Path, root: &Path) -> bool { path.starts_with(root) && path != root } +fn lexical(value: &str, cwd: &Path, work: &Path) -> Result { + if value.is_empty() || value.chars().any(|c| c < ' ' || "<>\"|?*".contains(c)) + || value.starts_with('\\') || value.starts_with('/') { return Err("unsafe path spelling".into()); } + let path = Path::new(value); + let absolute = path.is_absolute(); + if value.contains(':') && (!absolute || value[2..].contains(':')) { return Err("drive-relative/stream path forbidden".into()); } + let mut result = if absolute { PathBuf::new() } else { cwd.to_path_buf() }; + for component in path.components() { + match component { + Component::ParentDir => { + if absolute || result == work || !beneath(&result, work) { return Err("path traversal escaped work".into()); } + result.pop(); + }, + Component::Normal(name) => { + let name = name.to_str().ok_or("non-Unicode component")?; + if name.ends_with('.') || name.ends_with(' ') { return Err("ambiguous path component".into()); } + // Win32 reserves superscript port digits too, even with an extension. + // Trim the stem's spaces so an extension cannot hide a device alias. + let stem = name.split('.').next().unwrap_or("").trim_end_matches(' ').to_ascii_uppercase(); + let port = stem.strip_prefix("COM").or_else(|| stem.strip_prefix("LPT")); + if ["CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$", "CLOCK$"].contains(&stem.as_str()) + || matches!(port, Some("1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "¹" | "²" | "³")) { + return Err("reserved device component".into()); + } + result.push(name); + }, + Component::CurDir => (), + _ if absolute => result.push(component), + _ => return Err("unexpected path prefix".into()), + } + } + if !beneath(&result, work) { return Err("path outside work".into()); } + Ok(result) +} +fn relative(path: &Path, cwd: &Path, work: &Path) -> Result { + let target: Vec<_> = path.strip_prefix(work).map_err(|_| "object outside work")?.components().collect(); + let base: Vec<_> = cwd.strip_prefix(work).map_err(|_| "cwd outside work")?.components().collect(); + let shared = target.iter().zip(&base).take_while(|(a,b)| a == b).count(); + let mut result = PathBuf::new(); + for _ in shared..base.len() { result.push(".."); } + for component in &target[shared..] { result.push(component); } + if result.to_str().ok_or("non-Unicode relative object")?.starts_with(['@', '-']) { + return Err("relative object would become a librarian option/response file".into()); + } + if existing(&cwd.join(&result))? != path { return Err("relative object identity changed".into()); } + Ok(result) +} +fn run() -> Result { + let exe = existing(&env::current_exe().map_err(|e| e.to_string())?)?; + let tools = exe.parent().ok_or("missing tools parent")?; + let target = tools.parent().ok_or("missing target parent")?; + let work = target.parent().ok_or("missing work parent")?; + if exe.file_name().and_then(|s| s.to_str()) != Some("aware-lib.exe") + || tools.file_name().and_then(|s| s.to_str()) != Some("native-tools") + || target.file_name().and_then(|s| s.to_str()) != Some("cargo-target") { + return Err("adapter must occupy its fixed build location".into()); + } + let cwd = existing(&env::current_dir().map_err(|e| e.to_string())?)?; + if !beneath(&cwd, work) { return Err("cwd outside work".into()); } + let args: Vec = env::args_os().skip(1).map(|a| a.into_string().map_err(|_| "non-Unicode argument".to_string())).collect::>()?; + let mut output = None; let mut nologo = false; let mut brepro = false; + for arg in &args { + let lower = arg.to_ascii_lowercase(); + if lower.starts_with("-out:") || lower.starts_with("/out:") { + if output.is_some() { return Err("duplicate archive output".into()); } + let path = lexical(&arg[5..], &cwd, work)?; + let parent = existing(path.parent().ok_or("missing output parent")?)?; + let destination = parent.join(path.file_name().ok_or("missing output filename")?); + if !beneath(&destination, target) { return Err("output outside cargo target".into()); } + match fs::symlink_metadata(&destination) { + Ok(meta) if meta.file_attributes() & 0x400 != 0 || !meta.is_file() => return Err("invalid existing output".into()), + Ok(_) => (), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (), + Err(error) => return Err(error.to_string()), + } + if !matches!(destination.extension().and_then(|x| x.to_str()).map(str::to_ascii_lowercase).as_deref(), Some("a" | "lib")) { return Err("invalid archive extension".into()); } + output = Some(destination); + } else if lower == "-nologo" || lower == "/nologo" { + if nologo { return Err("duplicate nologo".into()); } nologo = true; + } else if lower == "-brepro" || lower == "/brepro" { + if brepro { return Err("duplicate Brepro".into()); } brepro = true; + } else if arg.starts_with(['-', '/', '@']) { return Err("unknown archive option/response file".into()); } + } + let output = output.ok_or("missing archive output")?; + let mut forwarded = Vec::new(); let mut appended = false; let mut objects = 0; + for arg in &args { + let lower = arg.to_ascii_lowercase(); + if lower.starts_with("-out:") || lower.starts_with("/out:") { forwarded.push(format!("/OUT:{}", output.display())); } + else if arg.starts_with(['-', '/']) { forwarded.push(arg.clone()); } + else { + let path = existing(&lexical(arg, &cwd, work)?)?; + if !beneath(&path, work) || !fs::metadata(&path).map_err(|e| e.to_string())?.is_file() { return Err("invalid archive input".into()); } + match path.extension().and_then(|x| x.to_str()).map(str::to_ascii_lowercase).as_deref() { + Some("o" | "obj") => { forwarded.push(relative(&path, &cwd, work)?.to_str().ok_or("non-Unicode relative object")?.to_string()); objects += 1; }, + Some("a" | "lib") if path == output && !appended && objects == 0 => { forwarded.push(path.to_str().ok_or("non-Unicode archive")?.to_string()); appended = true; }, + _ => return Err("extra archive or unsupported input".into()), + } + } + } + if objects == 0 { return Err("no object inputs".into()); } + if !brepro { forwarded.insert(0, "/Brepro".into()); } + let lib = existing(&work.join("compiler/msvc/bin/lib.exe"))?; + if lib != work.join("compiler/msvc/bin/lib.exe") { return Err("private librarian identity changed".into()); } + let status = Command::new(lib).args(forwarded).current_dir(cwd).status().map_err(|e| e.to_string())?; + status.code().ok_or_else(|| "librarian terminated without exit status".into()) +} +fn main() -> ExitCode { + match run() { + Ok(0) => ExitCode::SUCCESS, + Ok(code) => std::process::exit(code), + Err(error) => { eprintln!("AWARE_NATIVE_ARCHIVE_REFUSED: {error}"); ExitCode::from(2) }, + } +} +`; +export function prepareNativeArchiveAdapter(workRoot) { + const directory = join(workRoot, 'cargo-target', 'native-tools'); + mkdirSync(join(workRoot, 'cargo-target'), { recursive: true }); mkdirSync(directory); + const source = join(directory, 'aware-lib.rs'), executable = join(directory, 'aware-lib.exe'); + writeFileSync(source, NATIVE_ARCHIVE_ADAPTER_SOURCE, { flag: 'wx' }); + return { source, executable }; +} +export function nativeArchiveAdapterRecord(buildRoot) { + const path = 'work/cargo-target/native-tools/aware-lib.exe', file = join(buildRoot, ...path.split('/')); + return { path, size: lstatSync(file).size, sha256: fileDigest(file) }; +} +export function verifyNativeArchiveAdapter(record, report, buildRoot) { + same(record, nativeArchiveAdapterRecord(buildRoot), 'native archive adapter changed after its compilation'); + const expected = win32.resolve(buildRoot, record.path).toLowerCase(); + const images = report.images.filter(image => win32.resolve(image.path).toLowerCase() === expected); + assert.ok(images.length > 0, 'Cargo did not execute the bound native archive adapter'); + for (const image of images) { + assert.equal(image.kind, 'process'); assert.equal(image.size, record.size); + assert.equal(image.sha256, record.sha256, 'Cargo executed a different native archive adapter'); + } +} +const portable = path => path.split(sep).join('/'); +const same = (a, b, label) => assert.equal(canonicalJson(a), canonicalJson(b), label); +export function exactKeys(value, keys, label) { + assert.ok(value && typeof value === 'object' && !Array.isArray(value), `${label} must be an object`); + same(Object.keys(value).sort(), [...keys].sort(), `${label} has unexpected or missing keys`); +} +export const INPUT_IDS = Object.freeze(['aware-cargo-lock', 'reader-package-lock', 'builder-script', + 'compiler-closure-script', 'reader-settings-script', 'compiler-audit-script']); +export const NONCOMPILER_TOOL_IDS = Object.freeze(['git', 'node', 'npm-cli', 'postject', 'web-ifc-wasm', 'powershell']); +export const COMPILER_LAYOUT = Object.freeze({ + 'compiler-rust-bin': 'rust/bin', 'compiler-rust-lib': 'rust/lib', + 'compiler-msvc-bin': 'msvc/bin', 'compiler-msvc-include': 'msvc/include', 'compiler-msvc-lib': 'msvc/lib', + 'compiler-sdk-include': 'sdk/include', 'compiler-sdk-um-lib': 'sdk/um-lib', + 'compiler-sdk-ucrt-lib': 'sdk/ucrt-lib', 'compiler-sdk-bin': 'sdk/bin', +}); +export const COMPILER_IDS = Object.freeze(Object.keys(COMPILER_LAYOUT)); +export const CLOSURE_IDS = Object.freeze(['npm-cache', 'cargo-home', ...COMPILER_IDS]); +const tool = (closure, path) => Object.freeze({ closure, path }); +export const COMPILER_DESCRIPTOR = Object.freeze({ + schema: 'aware-windows-compiler/v1', layout: COMPILER_LAYOUT, + tools: Object.freeze({ cargo: tool('compiler-rust-bin', 'cargo.exe'), rustc: tool('compiler-rust-bin', 'rustc.exe'), + rustdoc: tool('compiler-rust-bin', 'rustdoc.exe'), cl: tool('compiler-msvc-bin', 'cl.exe'), + link: tool('compiler-msvc-bin', 'link.exe'), lib: tool('compiler-msvc-bin', 'lib.exe'), rc: tool('compiler-sdk-bin', 'rc.exe') }), + environment: Object.freeze({ PATH: ['msvc/bin', 'sdk/bin', 'rust/bin', ''], + INCLUDE: ['msvc/include', 'sdk/include/ucrt', 'sdk/include/shared', 'sdk/include/um', 'sdk/include/winrt', 'sdk/include/cppwinrt'], + LIB: ['msvc/lib', 'sdk/ucrt-lib', 'sdk/um-lib'], LIBPATH: ['msvc/lib', 'sdk/ucrt-lib', 'sdk/um-lib'], + // Disable the debugger-induced heap penalty without disabling DEBUG_PROCESS or image auditing. + PATHEXT: '.COM;.EXE;.BAT;.CMD', VSLANG: '1033', _NO_DEBUG_HEAP: '1', + VCINSTALLDIR: ['msvc'], VSCMD_ARG_TGT_ARCH: 'x64' }), + auditPolicy: 'aware-private-compiler-debug-events/v3', + startupPolicy: Object.freeze({ identity: 'aware-private-msvc-telemetry-denial/v1', + closure: 'compiler-msvc-bin', path: 'vctip.exe', exitCode: 0xe0000488 }), +}); +const REQUIRED = Object.freeze({ + 'compiler-rust-bin': [/^cargo\.exe$/, /^rustc\.exe$/, /^rustdoc\.exe$/, /^rustc_driver-[^/]+\.dll$/, /^std-[^/]+\.dll$/], + 'compiler-rust-lib': [/^rustlib\/x86_64-pc-windows-msvc\/lib\/libstd-[^/]+\.rlib$/, /^rustlib\/x86_64-pc-windows-msvc\/lib\/libcore-[^/]+\.rlib$/], + 'compiler-msvc-bin': [/^cl\.exe$/, /^link\.exe$/, /^lib\.exe$/, /^c1\.dll$/, /^c2\.dll$/, /^msvcp140\.dll$/, /^vcruntime140\.dll$/], + 'compiler-msvc-include': [/^vcruntime\.h$/], + 'compiler-msvc-lib': [/^libcmt\.lib$/, /^libvcruntime\.lib$/], + 'compiler-sdk-include': [/^ucrt\/stdio\.h$/, /^shared\/winerror\.h$/, /^um\/windows\.h$/, /^winrt\//, /^cppwinrt\//], + 'compiler-sdk-um-lib': [/^kernel32\.lib$/, /^user32\.lib$/], + 'compiler-sdk-ucrt-lib': [/^ucrt\.lib$/, /^libucrt\.lib$/], + 'compiler-sdk-bin': [/^rc\.exe$/, /^rcdll\.dll$/], +}); +export function validateRecordPath(path) { + assert.ok(typeof path === 'string' && path && !/[\\:<>"|?*\x00-\x1f\x7f]/.test(path), 'unsafe portable record path'); + for (const part of path.split('/')) { + assert.ok(part && part !== '.' && part !== '..' && !/[. ]$/.test(part) + && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part), 'unsafe portable record component'); + } +} +export function validateInventory(files, label) { + assert.ok(Array.isArray(files) && files.length, `${label} inventory must be nonempty`); + const seen = new Set(); let previous; + for (const record of files) { + exactKeys(record, ['path', 'size', 'sha256'], `${label} record`); validateRecordPath(record.path); + assert.ok(Number.isSafeInteger(record.size) && record.size >= 0 && SHA256.test(record.sha256), `${label} invalid file record`); + assert.ok(!seen.has(record.path.toLowerCase()), `${label} case-colliding file records`); seen.add(record.path.toLowerCase()); + if (previous != null) assert.ok(Buffer.compare(Buffer.from(previous), Buffer.from(record.path)) < 0, `${label} records are not sorted`); + previous = record.path; + } +} +export function validateCompilerManifest(manifest) { + exactKeys(manifest, ['schema', 'platform', 'arch', 'nodeVersion', 'rustVersion', 'target', 'source', 'settings', 'compiler', 'inputs', 'tools', 'closures'], 'Windows builder manifest'); + assert.equal(manifest.schema, 'aware-windows-repro-builder/v1'); + assert.equal(manifest.platform, 'win32'); assert.equal(manifest.arch, 'x64'); + assert.equal(manifest.nodeVersion, '24.14.0'); assert.equal(manifest.rustVersion, '1.95.0'); + assert.equal(manifest.target, 'x86_64-pc-windows-msvc'); + exactKeys(manifest.source, ['commit', 'tree', 'bundleSha256'], 'builder source'); + assert.ok(/^[0-9a-f]{40}$/.test(manifest.source.commit) && /^[0-9a-f]{40}$/.test(manifest.source.tree) && SHA256.test(manifest.source.bundleSha256), 'invalid builder source identity'); + exactKeys(manifest.inputs, INPUT_IDS, 'builder code/lock inputs'); + for (const value of Object.values(manifest.inputs)) assert.match(value, SHA256); + exactKeys(manifest.tools, NONCOMPILER_TOOL_IDS, 'noncompiler tool records'); + for (const id of NONCOMPILER_TOOL_IDS) { exactKeys(manifest.tools[id], ['id', 'sha256'], `${id} tool record`); assert.equal(manifest.tools[id].id, id); assert.match(manifest.tools[id].sha256, SHA256); } + same(manifest.compiler, COMPILER_DESCRIPTOR, 'compiler descriptor differs from the closed contract'); + exactKeys(manifest.closures, CLOSURE_IDS, 'compiler/dependency closures'); + for (const id of CLOSURE_IDS) { + exactKeys(manifest.closures[id], ['files'], `${id} closure`); + validateInventory(manifest.closures[id].files, id); + const names = manifest.closures[id].files.map(record => record.path.toLowerCase()); + for (const required of REQUIRED[id] ?? []) assert.ok(names.some(path => required.test(path)), `${id} lacks mandatory compiler support ${required}`); + } + const rust = manifest.closures['compiler-rust-bin'].files; + assert.equal(new Set(['cargo.exe', 'rustc.exe', 'rustdoc.exe'].map(name => rust.find(file => file.path.toLowerCase() === name)?.sha256)).size, + 3, 'Rust roles must identify distinct direct binaries, not rustup shims'); + same(manifest.settings, { bundle: { platform: 'node', format: 'cjs', target: 'node24' }, + sea: { disableExperimentalWarning: true, section: 'NODE_SEA_BLOB', sentinelFuse: 'NODE_SEA_FUSE_fce680ab2cc467b6e072b8b5df1996b2' } }, 'reader settings differ from compiler authority'); +} +export function validateWindowsPath(path, label = 'physical root', maximumLength = 200) { + assert.ok(typeof path === 'string' && /^[a-z]:[\\/]/i.test(path) && path.length <= maximumLength + && !/[;=<>"|?*\x00-\x1f\x7f]/.test(path) && !path.slice(2).includes(':'), `${label} must be a bounded local Windows drive path`); + for (const part of path.slice(3).split(/[\\/]/)) { + assert.ok(part && part !== '.' && part !== '..' && !/[. ]$/.test(part) + && !/^(con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/i.test(part), `${label} has an unsafe path component`); + } + return win32.normalize(path); +} +export function beneath(path, root) { + const child = resolve(path).toLowerCase(), parent = resolve(root).toLowerCase(); + return child === parent || child.startsWith(`${parent}${sep}`); +} +function checkedEntry(path, canonicalRoot) { + const stat = lstatSync(path); assert.ok(!stat.isSymbolicLink(), `path-redirection link refused: ${path}`); + assert.ok(beneath(realpathSync.native(path), canonicalRoot), `canonical path escaped closure: ${path}`); + return stat; +} +export function inventory(root) { + assert.ok(isAbsolute(root), 'closure root must be absolute'); + const stat = lstatSync(root); assert.ok(stat.isDirectory() && !stat.isSymbolicLink(), 'closure root must be a real directory'); + const files = [], canonicalRoot = realpathSync.native(root); + function walk(dir) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name), current = checkedEntry(path, canonicalRoot); + if (current.isDirectory()) walk(path); + else { assert.ok(current.isFile(), 'unsupported closure entry'); files.push({ path: portable(relative(root, path)), size: current.size, sha256: fileDigest(path) }); } + } + } + walk(root); files.sort((a, b) => Buffer.compare(Buffer.from(a.path), Buffer.from(b.path))); + validateInventory(files, root); return files; +} +export function copyDirectory(source, destination) { + const files = inventory(source); + assert.ok(!existsSync(destination), 'private copy destination must be fresh'); + mkdirSync(destination, { recursive: true }); + // Node 24.14.0 cpSync mangles Unicode directory names on Windows. Explicit + // file copies preserve the inventoried names and do not share mutable bytes. + for (const record of files) { + const output = join(destination, ...record.path.split('/')); mkdirSync(dirname(output), { recursive: true }); + copyFileSync(join(source, ...record.path.split('/')), output); + } + same(inventory(destination), files, 'private directory copy differs from its source inventory'); +} +export function validateCompilerLocator(locator) { + exactKeys(locator, ['schema', 'sourceBundle', 'tools', 'closures'], 'Windows builder locator'); + assert.equal(locator.schema, 'aware-windows-repro-locator/v1'); + exactKeys(locator.tools, NONCOMPILER_TOOL_IDS, 'noncompiler tool locator'); + exactKeys(locator.closures, CLOSURE_IDS, 'closure locator'); + for (const [id, path] of [...Object.entries(locator.tools), ...Object.entries(locator.closures), ['source bundle', locator.sourceBundle]]) validateWindowsPath(path, id); +} +export function compilerSummary(manifest) { + validateCompilerManifest(manifest); + return { schema: 'aware-compiler-authority/v1', descriptorSha256: digest(canonicalJson(COMPILER_DESCRIPTOR)), + auditPolicy: COMPILER_DESCRIPTOR.auditPolicy, + closures: Object.fromEntries(COMPILER_IDS.map(id => [id, digest(canonicalJson(manifest.closures[id].files))])) }; +} +export function materializeCompiler({ manifest, locator, workRoot, host }) { + validateCompilerManifest(manifest); validateCompilerLocator(locator); validateWindowsPath(workRoot, 'work root'); + const root = join(workRoot, 'compiler'); assert.ok(!existsSync(root), 'private compiler root must be fresh'); + const roots = {}; + for (const id of COMPILER_IDS) { + same(inventory(locator.closures[id]), manifest.closures[id].files, `${id} source inventory changed`); + const destination = join(root, ...COMPILER_LAYOUT[id].split('/')); mkdirSync(dirname(destination), { recursive: true }); + copyDirectory(locator.closures[id], destination); + // Bind the CANONICAL long path. The auditor validates every path it is given + // with .NET GetFullPath, which expands 8.3 short components once they exist + // on disk; join()/win32.resolve never do. A work root under a short temp + // path -- C:\Users\RUNNER~1 on a CI runner, or an AppData\Local\TEMP_~1 -- + // therefore produced a startup policy whose own path failed the auditor's + // self-consistency check ("Invalid private telemetry policy"), and left image + // attribution comparing a canonical observed path against a short root. + // Canonicalizing here is the single place that fixes both. + roots[id] = realpathSync.native(destination); + same(inventory(roots[id]), manifest.closures[id].files, `${id} private inventory changed`); + } + const canonicalRoot = realpathSync.native(root); + const tools = Object.fromEntries(Object.entries(COMPILER_DESCRIPTOR.tools).map(([id, role]) => [id, join(roots[role.closure], role.path)])); + const environment = Object.fromEntries(Object.entries(COMPILER_DESCRIPTOR.environment).map(([key, value]) => [key, + Array.isArray(value) ? value.map(path => path === '' ? host.system32 : join(canonicalRoot, ...path.split('/'))).join(';') : value])); + return Object.freeze({ root: canonicalRoot, roots: Object.freeze(roots), tools: Object.freeze(tools), environment: Object.freeze(environment), host, manifest }); +} +export function verifyPrivateCompiler(compiler) { + validateCompilerManifest(compiler.manifest); + for (const id of COMPILER_IDS) same(inventory(compiler.roots[id]), compiler.manifest.closures[id].files, `${id} private compiler changed`); +} +export function loaderObservedWindows(sharedObjects = process.report.getReport().sharedObjects) { + const locate = name => { + const matches = [...new Set(sharedObjects.filter(path => win32.basename(path).toLowerCase() === name).map(path => win32.normalize(path).toLowerCase()))]; + assert.equal(matches.length, 1, `expected one loader-observed ${name}`); return matches[0]; + }; + const kernel = win32.dirname(locate('kernel32.dll')), ntdll = win32.dirname(locate('ntdll.dll')); + assert.equal(kernel, ntdll, 'loader-observed Windows modules disagree'); assert.equal(win32.basename(kernel), 'system32'); + return { windows: win32.dirname(kernel), system32: kernel, powershell: win32.join(kernel, 'WindowsPowerShell', 'v1.0', 'powershell.exe') }; +} +export function systemEnvironment(host, tempRoot) { + return { SystemRoot: host.windows, WINDIR: host.windows, ComSpec: join(host.system32, 'cmd.exe'), + PATHEXT: COMPILER_DESCRIPTOR.environment.PATHEXT, PATH: host.system32, + PSModulePath: join(host.system32, 'WindowsPowerShell', 'v1.0', 'Modules'), TEMP: tempRoot, TMP: tempRoot }; +} +export function auditorTempParent(host) { + const parent = win32.join(host.windows, 'Temp'); + validateWindowsPath(parent, 'auditor temporary parent'); + assert.match(parent, /^[\x20-\x7e]+$/, 'inbox compiler auditor needs an ASCII Windows Temp path'); + const entry = lstatSync(parent); + assert.ok(entry.isDirectory() && !entry.isSymbolicLink(), 'auditor temporary parent must be a real directory'); + assert.equal(realpathSync.native(parent).toLowerCase(), parent.toLowerCase(), 'auditor temporary parent redirects elsewhere'); + return parent; +} +export function retainAuditorResult(requestPath, result, timeout) { + assert.match(requestPath, /-request\.local\.json$/); + const prefix = requestPath.replace(/-request\.local\.json$/, ''); + const stdout = result.stdout ?? Buffer.alloc(0), stderr = result.stderr ?? Buffer.alloc(0); + assert.ok(Buffer.isBuffer(stdout) && Buffer.isBuffer(stderr), 'auditor capture must preserve raw bytes'); + const text = `${stdout}${stderr}`, errors = []; + if (result.error || result.status !== 0) errors.push(new Error(`compiler auditor failed: ${result.error?.message ?? result.status}\n${text}`, { cause: result.error })); + const launch = { status: result.status ?? null, signal: result.signal ?? null, timeoutMs: timeout, + error: result.error ? { code: result.error.code ?? null, message: result.error.message } : null }; + for (const [suffix, bytes] of [['stdout.local.bin', stdout], ['stderr.local.bin', stderr], + ['command.local.log', text], ['launch.local.json', canonicalJson(launch)]]) { + try { writeFileSync(`${prefix}-${suffix}`, bytes, { flag: 'wx' }); } catch (error) { errors.push(error); } + } + if (errors.length > 1) throw new AggregateError(errors, 'compiler auditor execution or evidence persistence failed', { cause: errors[0] }); + if (errors.length) throw errors[0]; + return { text, stdout, stderr }; +} +function launchAuditor({ host, auditScript, auditDigest, request, requestPath, cwd, timeout = 120000 }) { + const bytes = readFileSync(auditScript); + assert.equal(digest(bytes), auditDigest, 'compiler auditor script changed before evaluation'); + writeFileSync(requestPath, canonicalJson(request), { flag: 'wx' }); + // Feed exact authenticated bytes through stdin. -File would re-read a mutable + // path; embedding the whole script in -EncodedCommand exceeds Windows argv limits. + const pathBytes = Buffer.from(requestPath, 'utf8').toString('base64'); + const command = `& ([ScriptBlock]::Create([Text.Encoding]::UTF8.GetString([Convert]::FromBase64String([Console]::In.ReadToEnd())))) -RequestPath ([Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${pathBytes}')))`; + const result = spawnSync(host.powershell, ['-NoProfile', '-NonInteractive', '-Command', command], + { cwd, env: systemEnvironment(host, auditorTempParent(host)), input: bytes.toString('base64'), encoding: null, windowsHide: true, timeout, maxBuffer: 128 * 1024 * 1024 }); + return retainAuditorResult(requestPath, result, timeout); +} +export function discoverSystemHost({ locator, manifest, auditScript, workRoot }) { + const observed = loaderObservedWindows(); + assert.equal(realpathSync.native(locator.tools.powershell).toLowerCase(), realpathSync.native(observed.powershell).toLowerCase(), 'only canonical inbox PowerShell is allowed'); + assert.equal(fileDigest(observed.powershell), manifest.tools.powershell.sha256, 'inbox PowerShell digest changed'); + assert.equal(fileDigest(auditScript), manifest.inputs['compiler-audit-script'], 'audit script digest changed'); + const cwd = join(workRoot, 'bootstrap'); mkdirSync(cwd, { recursive: true }); assert.equal(readdirSync(cwd).length, 0, 'bootstrap cwd must be empty'); + const output = join(workRoot, 'windows-host.local.json'); + launchAuditor({ host: observed, auditScript, auditDigest: manifest.inputs['compiler-audit-script'], request: { mode: 'host', output }, requestPath: join(workRoot, 'windows-host-request.local.json'), cwd }); + const actual = JSON.parse(readFileSync(output, 'utf8')); + exactKeys(actual, ['windows', 'system32'], 'Windows API host roots'); + for (const key of ['windows', 'system32']) assert.equal(realpathSync.native(actual[key]).toLowerCase(), realpathSync.native(observed[key]).toLowerCase(), `Windows API ${key} disagrees with loaded OS modules`); + return Object.freeze({ ...actual, powershell: observed.powershell }); +} +export function protectedWindowsPath(path, host) { + const value = win32.resolve(path); + if (win32.dirname(value).toLowerCase() === win32.resolve(host.system32).toLowerCase()) return true; + return ['WinSxS', 'Microsoft.NET/Framework', 'Microsoft.NET/Framework64', 'Microsoft.NET/assembly', 'System32/WindowsPowerShell'] + .some(part => windowsBeneath(value, win32.join(host.windows, ...part.split('/')))); +} +// Audit records always describe Windows paths, including when portable validation runs on Linux. +function windowsBeneath(path, root) { + const value = win32.relative(win32.resolve(root).toLowerCase(), win32.resolve(path).toLowerCase()); + return value !== '' && value !== '..' && !value.startsWith('..\\') && !win32.isAbsolute(value); +} +export function compilerStartupPolicy(compiler) { + const policy = COMPILER_DESCRIPTOR.startupPolicy; + const file = compiler.manifest.closures[policy.closure].files.find(file => file.path.toLowerCase() === policy.path); + return { identity: policy.identity, deniedImage: file ? { closure: policy.closure, relativePath: file.path, + path: win32.resolve(compiler.roots[policy.closure], file.path), size: file.size, sha256: file.sha256, exitCode: policy.exitCode } : null }; +} +export function verifyCompilerAudit(report, { compiler, targetRoot, toolPath }) { + exactKeys(report, ['schema', 'error', 'complete', 'exitCode', 'totalProcesses', 'activeProcesses', 'processes', 'images', 'identity', 'startupPolicy', 'eventCount'], 'compiler audit'); + assert.equal(report.schema, 'aware-compiler-debug-audit/v3'); assert.equal(report.complete, true, 'compiler audit is incomplete'); + const startupPolicy = compilerStartupPolicy(compiler), denied = startupPolicy.deniedImage; + same(report.startupPolicy, startupPolicy, 'compiler startup policy differs'); + assert.equal(report.error, null, 'compiler audit contains an error'); + assert.ok(Array.isArray(report.processes) && Array.isArray(report.images), 'compiler audit lists are missing'); + assert.ok(report.processes.length > 0 && report.images.length >= report.processes.length, 'compiler audit is empty'); + assert.equal(report.totalProcesses, report.processes.length, 'compiler audit missed a child process'); assert.equal(report.activeProcesses, 0); + assert.ok(Number.isSafeInteger(report.exitCode) && report.exitCode >= 0, 'compiler audit exit status is invalid'); + assert.ok(Number.isSafeInteger(report.eventCount) && report.eventCount > 0, 'invalid debugger event count'); + const instances = new Map(), pids = new Map(), events = new Set(); + const claimEvent = event => { + assert.ok(Number.isSafeInteger(event) && event > 0 && event <= report.eventCount && !events.has(event), 'invalid or repeated debugger event'); events.add(event); + }; + let previousStart = 0, lastExit = 0; + for (const process of report.processes) { + exactKeys(process, ['pid', 'path', 'exitCode', 'action', 'instance', 'startEvent', 'exitEvent'], 'audited process'); + assert.ok(Number.isSafeInteger(process.pid) && process.pid > 0 && process.pid <= 0xffffffff, 'invalid audited process PID'); + assert.equal(process.instance, instances.size + 1, 'invalid process instance order'); + claimEvent(process.startEvent); claimEvent(process.exitEvent); + assert.ok(process.startEvent > previousStart && process.exitEvent > process.startEvent, 'invalid process lifetime event order'); + assert.ok(!pids.has(process.pid) || pids.get(process.pid).exitEvent < process.startEvent, 'overlapping reused process PID'); + previousStart = process.startEvent; lastExit = Math.max(lastExit, process.exitEvent); + instances.set(process.instance, process); pids.set(process.pid, process); + assert.ok(Number.isSafeInteger(process.exitCode) && process.exitCode >= 0, 'audited process never exited'); + assert.equal(report.images.filter(image => image.instance === process.instance && image.kind === 'process').length, 1, 'audited process image is missing or repeated'); + const matches = denied && win32.resolve(process.path).toLowerCase() === denied.path.toLowerCase(); + assert.equal(process.action, matches ? 'blocked-telemetry' : 'observed', 'compiler process disposition differs'); + if (matches) { + assert.notEqual(process.instance, 1, 'root compiler cannot be blocked telemetry'); + assert.equal(process.exitCode, denied.exitCode, 'blocked telemetry exit status differs'); + const image = report.images.find(image => image.instance === process.instance && image.kind === 'process'); + assert.equal(image.sha256, denied.sha256, 'blocked telemetry digest differs'); + assert.equal(image.size, denied.size, 'blocked telemetry size differs'); + } + } + assert.equal(report.processes[0].startEvent, 1, 'root must be the first debugger event'); + assert.equal(lastExit, report.eventCount, 'completed audit must end at the final process exit'); + assert.equal(report.processes[0].exitCode, report.exitCode, 'root compiler exit status differs'); + assert.ok(toolPath && win32.resolve(report.processes[0].path).toLowerCase() === win32.resolve(toolPath).toLowerCase(), 'compiler audit root differs from the requested tool'); + same(report.identity, { source: compiler.manifest.source, buildId: digest(canonicalJson(compiler.manifest)), + auditScriptSha256: compiler.manifest.inputs['compiler-audit-script'] }, 'compiler audit identity differs'); + let previousImage = 0; + const classified = report.images.map(image => { + exactKeys(image, ['pid', 'path', 'kind', 'sha256', 'size', 'instance', 'event'], 'audited image'); + const process = instances.get(image.instance); + assert.ok(process && process.pid === image.pid && ['process', 'dll'].includes(image.kind), 'unclassified process/image lifetime'); + assert.ok(Number.isSafeInteger(image.event) && image.event > previousImage && image.event <= report.eventCount, 'invalid image event order'); + previousImage = image.event; + if (image.kind === 'process') { + assert.equal(image.event, process.startEvent, 'process image must identify its creation event'); + assert.equal(image.path, process.path, 'process image path differs'); + } else { + claimEvent(image.event); + assert.ok(image.event > process.startEvent && image.event < process.exitEvent, 'DLL image outside its process lifetime'); + } + validateWindowsPath(image.path, 'audited image', 32760); + assert.ok(Number.isSafeInteger(image.size) && image.size > 0 && SHA256.test(image.sha256), 'unhashed compiler image'); + if (protectedWindowsPath(image.path, compiler.host)) return { ...image, role: 'windows' }; + for (const id of COMPILER_IDS) if (windowsBeneath(image.path, compiler.roots[id])) { + const path = win32.relative(compiler.roots[id], image.path).replaceAll('\\', '/').toLowerCase(); + const expected = compiler.manifest.closures[id].files.find(file => file.path.toLowerCase() === path); + assert.ok(expected && expected.size === image.size && expected.sha256 === image.sha256, `unbound compiler image: ${path}`); + return { ...image, role: id }; + } + assert.ok(windowsBeneath(image.path, targetRoot) && /\.(exe|dll)$/i.test(image.path), `compiler loaded an image outside its authority: ${image.path}`); + return { ...image, role: 'derived-cargo-output' }; + }); + return { ...report, images: classified }; +} +export function runAuditedCompiler({ compiler, toolPath, args, cwd, env, auditScript, evidenceRoot, label, targetRoot, timeout = 5400000 }) { + // Windows environment names are case-insensitive. Reject ambiguous spellings before any launch. + const heapKeys = Object.keys(env ?? {}).filter(key => key.toUpperCase() === '_NO_DEBUG_HEAP'); + assert.ok(heapKeys.length === 1 && heapKeys[0] === '_NO_DEBUG_HEAP' && env._NO_DEBUG_HEAP === '1', + 'audited compiler requires the fixed _NO_DEBUG_HEAP=1 environment'); + verifyPrivateCompiler(compiler); + assert.equal(fileDigest(auditScript), compiler.manifest.inputs['compiler-audit-script'], 'compiler auditor script changed'); + assert.ok(Object.values(compiler.tools).some(path => resolve(path).toLowerCase() === resolve(toolPath).toLowerCase()), 'audited command is not a private compiler role'); + assert.match(label, /^[a-z0-9-]+$/); mkdirSync(evidenceRoot, { recursive: true }); + const output = join(evidenceRoot, `${label}-audit.local.json`); + const request = { mode: 'run', output, executable: toolPath, args, cwd, environment: env, timeoutMs: timeout, + startupPolicy: compilerStartupPolicy(compiler), + windows: compiler.host.windows, system32: compiler.host.system32, + identity: { source: compiler.manifest.source, buildId: digest(canonicalJson(compiler.manifest)), auditScriptSha256: compiler.manifest.inputs['compiler-audit-script'] } }; + const captured = launchAuditor({ host: compiler.host, auditScript, auditDigest: compiler.manifest.inputs['compiler-audit-script'], request, requestPath: join(evidenceRoot, `${label}-request.local.json`), cwd: join(dirname(compiler.root), 'bootstrap'), timeout: timeout + 30000 }); + const { text } = captured; + const report = verifyCompilerAudit(JSON.parse(readFileSync(output, 'utf8')), { compiler, targetRoot, toolPath }); + assert.equal(report.exitCode, 0, `private compiler failed: ${text}`); + return { ...captured, report, evidencePath: output, evidenceSha256: fileDigest(output) }; +} + +// Outer A/B acceptance must validate each complete process audit, not just compare artifact bytes. +export function verifyCompilerProvenance({ buildRoot, manifest, host = loaderObservedWindows() }) { + validateCompilerManifest(manifest); + const read = path => JSON.parse(readFileSync(path, 'utf8')); + const record = read(join(buildRoot, 'evidence', 'compiler-provenance.json')); + exactKeys(record, ['schema', 'source', 'buildId', 'compiler', 'audits', 'artifacts', 'nativeArchiveAdapter'], 'compiler provenance'); + assert.equal(record.schema, 'aware-compiler-provenance/v1'); + same(record.source, manifest.source, 'compiler provenance source differs'); + assert.equal(record.buildId, digest(canonicalJson(manifest)), 'compiler provenance build differs'); + same(record.compiler, compilerSummary(manifest), 'compiler provenance authority differs'); + same(record.artifacts, inventory(join(buildRoot, 'artifacts')), 'compiler provenance artifact inventory differs'); + const root = join(buildRoot, 'work', 'compiler'); + const roots = Object.fromEntries(COMPILER_IDS.map(id => [id, join(root, ...COMPILER_LAYOUT[id].split('/'))])); + const compiler = { root, roots, host, manifest }; + verifyPrivateCompiler(compiler); + const labels = ['cargo-version', 'rust-version', 'rust-sysroot', 'rust-target-libdir', 'native-archive-adapter-build', 'cargo-build']; + assert.ok(Array.isArray(record.audits) && record.audits.length === labels.length, 'compiler provenance audit set is incomplete'); + same(record.audits.map(audit => audit.label).sort(), [...labels].sort(), 'compiler provenance audit labels differ'); + for (const audit of record.audits) { + exactKeys(audit, ['label', 'path', 'sha256'], 'compiler provenance audit reference'); + assert.equal(audit.path, `evidence/${audit.label}-audit.local.json`); + assert.equal(fileDigest(join(buildRoot, ...audit.path.split('/'))), audit.sha256, 'retained compiler audit digest differs'); + const role = audit.label.startsWith('cargo-') ? COMPILER_DESCRIPTOR.tools.cargo : COMPILER_DESCRIPTOR.tools.rustc; + const toolPath = join(roots[role.closure], role.path); + const report = verifyCompilerAudit(read(join(buildRoot, ...audit.path.split('/'))), { compiler, toolPath, targetRoot: join(buildRoot, 'work', 'cargo-target') }); + assert.equal(report.exitCode, 0, 'retained compiler failed'); + if (audit.label === 'cargo-build') verifyNativeArchiveAdapter(record.nativeArchiveAdapter, report, buildRoot); + } + return { sha256: fileDigest(join(buildRoot, 'evidence', 'compiler-provenance.json')), audits: record.audits }; +} diff --git a/cli/windows-compiler-closure.test.mjs b/cli/windows-compiler-closure.test.mjs new file mode 100644 index 000000000..6da58eace --- /dev/null +++ b/cli/windows-compiler-closure.test.mjs @@ -0,0 +1,387 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync, existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; +import { compilerFixture, COMPILER_FIXTURE_FILES } from './windows-compiler-fixture.mjs'; +import { createWindowsBuilderRecords } from './create-windows-internal-repro-inputs.mjs'; +import { verifyNativeIncludes, verifyNativeLinkInputs } from './windows-compiler-native-fixture.mjs'; +import { COMPILER_IDS, COMPILER_LAYOUT, COMPILER_DESCRIPTOR, compilerStartupPolicy, canonicalJson, digest, fileDigest, inventory, copyDirectory, loaderObservedWindows, verifyCompilerAudit, + protectedWindowsPath, retainAuditorResult, runAuditedCompiler, validateCompilerLocator, validateCompilerManifest, validateInventory, validateRecordPath, validateWindowsPath, + prepareNativeArchiveAdapter, nativeArchiveAdapterRecord, verifyNativeArchiveAdapter } from './windows-compiler-closure.mjs'; +import { loadVerifiedBuildModules, runningInputFiles, rejectedAmbientKeys, validateBootstrapLocator, verifyBuildAuthority, + bootstrapSystemEnvironment, verifyConsumedClosure } from './build-windows-internal-repro.mjs'; + +test('every observed native adapter image must match its original compiled bytes', () => { + const fixture=compilerFixture(),work=join(fixture.root,'work'); + try { + mkdirSync(work); const adapter=prepareNativeArchiveAdapter(work); writeFileSync(adapter.executable,'compiled trusted adapter'); + const record=nativeArchiveAdapterRecord(fixture.root); + const image={path:adapter.executable,kind:'process',size:record.size,sha256:record.sha256}; + assert.doesNotThrow(()=>verifyNativeArchiveAdapter(record,{images:[image,{...image}]},fixture.root)); + for(const images of [[],[{...image,sha256:'f'.repeat(64)}],[{...image,size:record.size+1}],[image,{...image,sha256:'f'.repeat(64)}]]) { + assert.throws(()=>verifyNativeArchiveAdapter(record,{images},fixture.root)); + } + writeFileSync(adapter.executable,'changed later adapter'); + assert.throws(()=>verifyNativeArchiveAdapter(record,{images:[image]},fixture.root),/changed after its compilation/); + } finally {rmSync(fixture.root,{recursive:true,force:true});} +}); + +test('structured native include proof preserves Unicode and refuses foreign or missing inputs', () => { + const source = 'C:\\build Łódź 😀\\native.c', roots = ['C:\\private Łódź 😀\\sdk', 'C:\\private Łódź 😀\\msvc']; + const report = { Version: '1.2', Data: { Source: source.toLowerCase(), ProvidedModule: '', Includes: [roots[0] + '\\um\\windows.h', roots[1] + '\\stdio.h'] } }; + assert.equal(verifyNativeIncludes(report, source, roots).length, 2); + for (const modify of [ + copy => { copy.Version = 'unknown'; }, + copy => { copy.Data.Source = 'C:\\foreign\\native.c'; }, + copy => { copy.Data.Includes = []; }, + copy => { copy.Data.Includes.pop(); }, + copy => { copy.Data.Includes.push('C:\\private Łódź 😀\\sdk-other\\foreign.h'); }, + copy => { copy.Data.Includes.push('C:\\private Łódź 😀\\sdk\\..\\foreign.h'); }, + copy => { copy.Data.Includes.push('relative.h'); }, + ]) { const changed = structuredClone(report); modify(changed); assert.throws(() => verifyNativeIncludes(changed, source, roots)); } +}); + +test('auditor diagnostics retain real nonzero, startup and timeout results before throwing', () => { + const fixture = compilerFixture(); + try { + const samples = [ + ['exit', spawnSync(process.execPath, ['-e', 'process.stdout.write(Buffer.from([0,255,1]));process.stderr.write(Buffer.from([254,0,2]));process.exitCode=7'], { encoding: null, windowsHide: true }), undefined], + ['startup', spawnSync(join(fixture.root, 'missing-executable'), [], { encoding: null, windowsHide: true }), 'ENOENT'], + ['timeout', spawnSync(process.execPath, ['-e', 'setTimeout(()=>{},30000)'], { encoding: null, windowsHide: true, timeout: 100 }), 'ETIMEDOUT'], + ]; + for (const [label, result, code] of samples) { + if (code) assert.equal(result.error?.code, code); else assert.equal(result.status, 7); + const prefix = join(fixture.root, label), request = `${prefix}-request.local.json`; + writeFileSync(request, '{}', { flag: 'wx' }); + assert.throws(() => retainAuditorResult(request, result, 100), error => { + assert.match(error.message, /compiler auditor failed/); + assert.equal(error.cause, result.error); return true; + }); + assert.deepEqual(readFileSync(`${prefix}-stdout.local.bin`), result.stdout ?? Buffer.alloc(0)); + assert.deepEqual(readFileSync(`${prefix}-stderr.local.bin`), result.stderr ?? Buffer.alloc(0)); + const launch = JSON.parse(readFileSync(`${prefix}-launch.local.json`, 'utf8')); + assert.equal(launch.error?.code, code); assert.equal(launch.status, result.status ?? null); + assert.equal(launch.signal, result.signal ?? null); assert.equal(launch.timeoutMs, 100); + } + const result = samples[0][1], prefix = join(fixture.root, 'collision'); + writeFileSync(`${prefix}-stdout.local.bin`, 'preserve'); + assert.throws(() => retainAuditorResult(`${prefix}-request.local.json`, result, 100), error => { + assert.ok(error instanceof AggregateError); assert.equal(error.errors.length, 2); + assert.match(error.errors[0].message, /compiler auditor failed/); assert.equal(error.errors[1].code, 'EEXIST'); return true; + }); + assert.equal(readFileSync(`${prefix}-stdout.local.bin`, 'utf8'), 'preserve'); + assert.deepEqual(readFileSync(`${prefix}-stderr.local.bin`), result.stderr); + assert.ok(existsSync(`${prefix}-command.local.log`) && existsSync(`${prefix}-launch.local.json`)); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('native link capture binds every library and object to actual declared bytes', () => { + const fixture = compilerFixture(); + try { + const captured = join(fixture.root, 'link-repro'), object = join(fixture.root, 'native.obj'); + mkdirSync(captured); writeFileSync(object, 'actual object'); + for (const [name, content] of Object.entries({ 'native.obj': 'actual object', 'env.setting': 'LIB=private', 'link.rsp': 'native.obj', 'kernel32.lib': 'actual SDK', 'LIBCMT.lib': 'actual CRT' })) writeFileSync(join(captured, name), content); + const files = inventory(captured); + const compiler = { manifest: { closures: { 'compiler-msvc-lib': { files: files.filter(file => file.path === 'LIBCMT.lib') }, 'compiler-sdk-um-lib': { files: files.filter(file => file.path === 'kernel32.lib') }, 'compiler-sdk-ucrt-lib': { files: [] } } } }; + assert.equal(verifyNativeLinkInputs(captured, compiler, object).length, 5); + for (const name of ['native.obj', 'kernel32.lib', 'LIBCMT.lib']) { + const path = join(captured, name), original = readFileSync(path); + writeFileSync(path, 'mutated'); assert.throws(() => verifyNativeLinkInputs(captured, compiler, object)); writeFileSync(path, original); + rmSync(path); assert.throws(() => verifyNativeLinkInputs(captured, compiler, object)); writeFileSync(path, original); + } + for (const name of ['foreign.lib', 'foreign.obj']) { + const path = join(captured, name); writeFileSync(path, 'unbound'); + assert.throws(() => verifyNativeLinkInputs(captured, compiler, object), /unbound/); rmSync(path); + } + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('compiler authority is independent of physical source locations', () => { + const a = compilerFixture(), b = compilerFixture(); + try { + const first = createWindowsBuilderRecords(a.input), second = createWindowsBuilderRecords(b.input); + assert.equal(first.manifestText, second.manifestText); assert.equal(first.buildId, second.buildId); + assert.equal(Object.keys(first.manifest.closures).length, 11); + assert.equal(Object.keys(first.manifest.inputs).length, 6); + assert.equal('environment' in first.locator, false); + assert.equal('cargo' in first.locator.tools, false); + assert.doesNotThrow(() => validateCompilerManifest(first.manifest)); + } finally { rmSync(a.root, { recursive: true, force: true }); rmSync(b.root, { recursive: true, force: true }); } +}); + +test('private copies preserve real Unicode names and are independent of original bytes', () => { + const fixture = compilerFixture(); + try { + const source = fixture.input.closures['compiler-msvc-include'], destination = join(fixture.root, 'Łódź with spaces', 'private'); + writeFileSync(join(source, 'żółć.h'), 'Unicode header'); copyDirectory(source, destination); + assert.equal(readFileSync(join(destination, 'żółć.h'), 'utf8'), 'Unicode header'); + assert.equal(canonicalJson(inventory(source)), canonicalJson(inventory(destination))); + writeFileSync(join(source, 'żółć.h'), 'changed original'); + assert.equal(readFileSync(join(destination, 'żółć.h'), 'utf8'), 'Unicode header'); + assert.throws(() => copyDirectory(source, destination), /fresh/); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('self-consistent manifests cannot drop mandatory compiler inputs', () => { + const fixture = compilerFixture(); + try { + const original = createWindowsBuilderRecords(fixture.input).manifest; + for (const id of COMPILER_IDS) for (const path of COMPILER_FIXTURE_FILES[id]) { + const changed = structuredClone(original); + changed.closures[id].files = changed.closures[id].files.filter(file => file.path !== path); + assert.throws(() => validateCompilerManifest(changed), /mandatory compiler support|nonempty/, `${id}/${path}`); + } + const shims = structuredClone(original); + for (const file of shims.closures['compiler-rust-bin'].files.filter(file => /^(cargo|rustc|rustdoc)\.exe$/.test(file.path))) file.sha256 = 'e'.repeat(64); + assert.throws(() => validateCompilerManifest(shims), /rustup shims/); + const reordered = structuredClone(original); reordered.compiler.environment.LIB.reverse(); + assert.throws(() => validateCompilerManifest(reordered), /descriptor differs/); + const oldPolicy = structuredClone(original); oldPolicy.compiler.auditPolicy = 'aware-private-compiler-debug-events/v2'; + assert.throws(() => validateCompilerManifest(oldPolicy), /descriptor differs/); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('portable records reject unsafe aliases, case collisions and unsorted input', () => { + for (const path of ['../x', 'a/../x', '/x', 'a//x', 'a\\x', 'c:x', 'x.', 'x ', 'NUL.exe', 'a/COM1', 'a?b', 'a*b', 'a validateRecordPath(path), /unsafe/); + const records = [{ path: 'A.h', size: 1, sha256: 'a'.repeat(64) }, { path: 'a.h', size: 1, sha256: 'b'.repeat(64) }]; + assert.throws(() => validateInventory(records, 'fixture'), /case-colliding/); + assert.throws(() => validateInventory([{ ...records[0], path: 'z.h' }, { ...records[1], path: 'b.h' }], 'fixture'), /not sorted/); +}); + +test('physical compiler roots have explicit Windows syntax and length limits', () => { + for (const root of ['C:\\build with spaces\\żółć', `D:\\${'x'.repeat(180)}`]) assert.doesNotThrow(() => validateWindowsPath(root)); + for (const root of ['C:relative', '\\\\server\\share', '\\\\?\\C:\\work', 'C:\\bad;path', 'C:\\bad=path', 'C:\\bad\npath', 'C:\\bad.', 'C:\\..\\escape', `C:\\${'x'.repeat(201)}`]) assert.throws(() => validateWindowsPath(root)); +}); + +test('compiler locator refuses environment authority and redundant compiler paths', () => { + const fixture = compilerFixture(); + try { + const locator = createWindowsBuilderRecords(fixture.input).locator; + assert.throws(() => validateCompilerLocator({ ...locator, environment: { PATH: 'C:\\hostile' } }), /unexpected or missing keys/); + assert.throws(() => validateCompilerLocator({ ...locator, tools: { ...locator.tools, cargo: 'C:\\hostile\\cargo.exe' } }), /unexpected or missing keys/); + assert.throws(() => createWindowsBuilderRecords({ ...fixture.input, environment: {} }), /unexpected or missing keys/); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('every locator path is rejected syntactically before any filesystem lookup', () => { + const fixture = compilerFixture(); + try { + const { manifest } = createWindowsBuilderRecords(fixture.input); + const locator = { schema: 'aware-windows-repro-locator/v1', sourceBundle: 'C:/unread/source.bundle', + tools: Object.fromEntries(Object.keys(fixture.input.tools).map(id => [id, `C:/unread/${id}`])), + closures: Object.fromEntries(Object.keys(fixture.input.closures).map(id => [id, `C:/unread/${id}`])) }; + for (const family of ['tools', 'closures']) for (const id of Object.keys(locator[family])) { + const bad = structuredClone(locator); bad[family][id] = '\\\\must-not-contact.invalid\\share'; + assert.throws(() => verifyBuildAuthority({ manifest, locator: bad, env: {} }), /unsafe bootstrap/); + } + assert.throws(() => verifyBuildAuthority({ manifest, locator: { ...locator, sourceBundle: '\\\\?\\C:\\device' }, env: {} }), /unsafe bootstrap/); + assert.doesNotThrow(() => validateBootstrapLocator(locator)); + const env = bootstrapSystemEnvironment('C:/temp', ['C:\\Windows\\System32\\kernel32.dll', 'C:\\Windows\\System32\\ntdll.dll']); + assert.equal(env.SystemRoot, 'c:\\windows'); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('private npm cache mutation is refused at the post-consumption boundary', () => { + const fixture = compilerFixture(); + try { + const { manifest } = createWindowsBuilderRecords(fixture.input), root = fixture.input.closures['npm-cache']; + assert.equal(verifyConsumedClosure('npm-cache', root, manifest, inventory), root); + writeFileSync(join(root, 'new-npm-state'), 'mutation'); + assert.throws(() => verifyConsumedClosure('npm-cache', root, manifest, inventory), /inventory mismatch/); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('ambient compiler overrides are rejected case-insensitively', () => { + const poisoned = { Path: 'x', RustFlags: 'x', _cl_: 'x', RuStUp_HoMe: 'x', cArGo_HoMe: 'x', CC_x86_64_pc_windows_msvc: 'x', WindowsSdkDir: 'x', VCToolsInstallDir: 'x', vcInstallDir: 'x', vScMd_ArG_TgT_ArCh: 'x64', _nO_dEbUg_HeAp: '1' }; + assert.deepEqual(rejectedAmbientKeys(poisoned), Object.keys(poisoned).sort()); + assert.deepEqual(rejectedAmbientKeys({ SystemRoot: 'a', systemroot: 'b' }), ['SystemRoot', 'systemroot']); +}); + +test('compiler heap setting is fixed manifest authority and rejects alternate spellings', () => { + const fixture = compilerFixture(); + try { + const { manifest } = createWindowsBuilderRecords(fixture.input); + assert.equal(manifest.compiler.environment._NO_DEBUG_HEAP, '1'); + for (const value of [undefined, '0', 1]) { + const changed = structuredClone(manifest); + if (value === undefined) delete changed.compiler.environment._NO_DEBUG_HEAP; + else changed.compiler.environment._NO_DEBUG_HEAP = value; + assert.throws(() => validateCompilerManifest(changed), /descriptor differs/); + } + for (const duplicate of [false, true]) { + const changed = structuredClone(manifest); + changed.compiler.environment._no_debug_heap = '1'; + if (!duplicate) delete changed.compiler.environment._NO_DEBUG_HEAP; + assert.throws(() => validateCompilerManifest(changed), /descriptor differs/); + } + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('auditor rejects missing, altered or ambiguous heap mode before filesystem access', () => { + for (const env of [undefined, {}, { _NO_DEBUG_HEAP: '0' }, { _NO_DEBUG_HEAP: 1 }, + { _no_debug_heap: '1' }, { _NO_DEBUG_HEAP: '1', _no_debug_heap: '0' }]) { + // No compiler or paths exist: a bypass would reach private-compiler verification and fail differently. + assert.throws(() => runAuditedCompiler({ env }), /fixed _NO_DEBUG_HEAP=1/); + } +}); + +test('MSVC discovery markers are fixed descriptor authority, never host inputs', () => { + const fixture = compilerFixture(); + try { + const { manifest } = createWindowsBuilderRecords(fixture.input); + assert.deepEqual(manifest.compiler.environment.VCINSTALLDIR, ['msvc']); + assert.equal(manifest.compiler.environment.VSCMD_ARG_TGT_ARCH, 'x64'); + for (const key of ['VCINSTALLDIR', 'VSCMD_ARG_TGT_ARCH']) { + const missing = structuredClone(manifest); delete missing.compiler.environment[key]; + assert.throws(() => validateCompilerManifest(missing), /descriptor differs/); + const changed = structuredClone(manifest); changed.compiler.environment[key] = key === 'VCINSTALLDIR' ? ['C:/hostile'] : 'x86'; + assert.throws(() => validateCompilerManifest(changed), /descriptor differs/); + } + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('inventory rejects redirected roots and nested directories on the real filesystem', () => { + const fixture = compilerFixture(); + try { + const redirected = join(fixture.root, 'redirected'); + symlinkSync(fixture.input.closures['compiler-msvc-include'], redirected, process.platform === 'win32' ? 'junction' : 'dir'); + assert.throws(() => inventory(redirected), /real directory/); + const nested = join(fixture.input.closures['compiler-rust-bin'], 'redirected'); + symlinkSync(fixture.input.closures['compiler-msvc-include'], nested, process.platform === 'win32' ? 'junction' : 'dir'); + assert.throws(() => inventory(fixture.input.closures['compiler-rust-bin']), /path-redirection/); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('a wrong-hash helper is refused before top-level code can execute', async () => { + const fixture = compilerFixture(); + try { + const builder = join(fixture.root, 'running', 'cli', 'build-windows-internal-repro.mjs'); + const files = runningInputFiles(builder), marker = join(fixture.root, 'executed-marker'); + for (const [id, path] of Object.entries(files)) { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, id); } + const inputs = Object.fromEntries(Object.entries(files).map(([id, path]) => [id, fileDigest(path)])); + writeFileSync(files['compiler-closure-script'], `import {writeFileSync} from 'node:fs'; writeFileSync(${JSON.stringify(marker)}, 'executed');`); + await assert.rejects(loadVerifiedBuildModules(inputs, files), /running compiler-closure-script differs/); + assert.equal(existsSync(marker), false); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('manifest-valid helpers cannot execute before matching the extracted source', async () => { + const fixture = compilerFixture(); + try { + const source = join(fixture.root, 'extracted'), running = join(fixture.root, 'runner', 'cli', 'build-windows-internal-repro.mjs'); + const files = runningInputFiles(running), extracted = runningInputFiles(join(source, 'cli', 'build-windows-internal-repro.mjs')); + const marker = join(fixture.root, 'source-mismatch-executed'); + for (const map of [files, extracted]) for (const [id, path] of Object.entries(map)) { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, id); } + writeFileSync(files['compiler-closure-script'], `import {writeFileSync} from 'node:fs'; writeFileSync(${JSON.stringify(marker)}, 'executed');`); + writeFileSync(join(source, 'cli', 'Cargo.lock'), 'lock'); writeFileSync(join(source, 'cli-connection-reader', 'package-lock.json'), '{}'); + const inputs = { 'aware-cargo-lock': fileDigest(join(source, 'cli', 'Cargo.lock')), 'reader-package-lock': fileDigest(join(source, 'cli-connection-reader', 'package-lock.json')), + ...Object.fromEntries(Object.entries(files).map(([id, path]) => [id, fileDigest(path)])) }; + await assert.rejects(loadVerifiedBuildModules(inputs, files, source), /extracted source/); assert.equal(existsSync(marker), false); + // Source-equal minimal modules are evaluated from their authenticated bytes. + for (const map of [files, extracted]) { + writeFileSync(map['compiler-closure-script'], 'export const authenticated = true;'); + writeFileSync(map['reader-settings-script'], 'export const READER_BUILD_SETTINGS = {authenticated:true};'); + } + for (const [id, path] of Object.entries(files)) inputs[id] = fileDigest(path); + const loaded = await loadVerifiedBuildModules(inputs, files, source); + assert.equal(loaded.compiler.authenticated, true); assert.equal(loaded.settings.authenticated, true); + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); + +test('OS discovery uses loaded Windows modules and excludes writable Windows paths', () => { + const host = loaderObservedWindows(['C:\\Windows\\System32\\KERNEL32.DLL', 'C:\\Windows\\SYSTEM32\\ntdll.dll']); + assert.equal(host.powershell.toLowerCase(), 'c:\\windows\\system32\\windowspowershell\\v1.0\\powershell.exe'); + assert.throws(() => loaderObservedWindows(['C:\\fake\\kernel32.dll', 'C:\\Windows\\System32\\ntdll.dll']), /disagree/); + // Native filesystem path semantics here; protectedWindowsPath never accepts a Windows-root prefix alone. + const nativeHost = { windows: join(process.cwd(), 'Windows'), system32: join(process.cwd(), 'Windows', 'System32') }; + assert.equal(protectedWindowsPath(join(nativeHost.system32, 'kernel32.dll'), nativeHost), true); + assert.equal(protectedWindowsPath(join(nativeHost.windows, 'Temp', 'evil.dll'), nativeHost), false); + assert.equal(protectedWindowsPath(join(nativeHost.system32, 'Temp', 'evil.dll'), nativeHost), false); + assert.equal(protectedWindowsPath(join(nativeHost.windows, 'Microsoft.NET', 'Framework64', 'v4.0.30319', 'clr.dll'), nativeHost), true); +}); + +test('compiler audit requires complete bound process/image evidence and the requested root tool', () => { + const fixture = compilerFixture(); + try { + writeFileSync(join(fixture.input.closures['compiler-msvc-bin'], 'vctip.exe'), 'private telemetry image'); + const manifest = createWindowsBuilderRecords(fixture.input).manifest; + const roots = Object.fromEntries(COMPILER_IDS.map(id => [id, join('C:/private/compiler', COMPILER_LAYOUT[id])])); + const toolPath = join(roots['compiler-rust-bin'], 'rustc.exe'); + const rustc = manifest.closures['compiler-rust-bin'].files.find(file => file.path === 'rustc.exe'); + const compiler = { manifest, roots, host: { windows: 'C:/Windows', system32: 'C:/Windows/System32' } }; + const options = { compiler, toolPath, targetRoot: 'C:/private/cargo-target' }; + const report = { schema: 'aware-compiler-debug-audit/v3', eventCount: 10, startupPolicy: compilerStartupPolicy(compiler), error: null, complete: true, exitCode: 0, totalProcesses: 1, activeProcesses: 0, + processes: [{ pid: 42, instance: 1, startEvent: 1, exitEvent: 10, path: toolPath, exitCode: 0, action: 'observed' }], + images: [{ pid: 42, instance: 1, event: 1, path: toolPath, kind: 'process', sha256: rustc.sha256, size: rustc.size }], + identity: { source: manifest.source, buildId: digest(canonicalJson(manifest)), auditScriptSha256: manifest.inputs['compiler-audit-script'] } }; + assert.equal(verifyCompilerAudit(report, options).images[0].role, 'compiler-rust-bin'); + const omittedPolicy = structuredClone(report); delete omittedPolicy.startupPolicy; + assert.throws(() => verifyCompilerAudit(omittedPolicy, options), /missing keys/); + for (const policy of [{ ...report.startupPolicy, deniedImage: null }, { ...report.startupPolicy, identity: 'unarmed' }]) { + assert.throws(() => verifyCompilerAudit({ ...report, startupPolicy: policy }, options), /startup policy differs/); + } + const denied = report.startupPolicy.deniedImage; + const blocked = { ...structuredClone(report), totalProcesses: 2, + processes: [...report.processes, { pid: 43, instance: 2, startEvent: 3, exitEvent: 7, path: denied.path, exitCode: denied.exitCode, action: 'blocked-telemetry' }], + images: [...report.images, { pid: 43, instance: 2, event: 3, path: denied.path, kind: 'process', size: denied.size, sha256: denied.sha256 }] }; + assert.doesNotThrow(() => verifyCompilerAudit(blocked, options)); + for (const mutate of [ + r => { r.processes[1].action = 'observed'; }, r => { r.processes[1].exitCode = 0; }, + r => { r.images[1].sha256 = 'f'.repeat(64); }, r => { r.images[1].size++; }, + r => { r.processes[1].path = r.images[1].path = join(roots['compiler-msvc-bin'], 'link.exe'); }, + r => { r.processes.pop(); }, r => { r.images.pop(); }, r => { r.activeProcesses = 1; }, + r => { r.processes[0].action = 'blocked-telemetry'; }, + ]) { const changed = structuredClone(blocked); mutate(changed); assert.throws(() => verifyCompilerAudit(changed, options)); } + const noTelemetryCompiler = structuredClone(compiler); + noTelemetryCompiler.manifest.closures['compiler-msvc-bin'].files = noTelemetryCompiler.manifest.closures['compiler-msvc-bin'].files.filter(file => file.path !== 'vctip.exe'); + assert.equal(compilerStartupPolicy(noTelemetryCompiler).deniedImage, null); + for (const changed of [{ ...report, complete: false }, { ...report, error: 'lost event' }, { ...report, activeProcesses: 1 }, + { ...report, totalProcesses: 2 }, { ...report, processes: [{ ...report.processes[0], exitCode: null }] }, + { ...report, images: [{ ...report.images[0], sha256: 'f'.repeat(64) }] }]) assert.throws(() => verifyCompilerAudit(changed, options)); + assert.throws(() => verifyCompilerAudit(report, { ...options, toolPath: join(roots['compiler-rust-bin'], 'cargo.exe') }), /requested tool/); + const outside = { ...report, images: [...report.images, { ...report.images[0], event: 2, kind: 'dll', path: 'C:/Windows/Temp/evil.dll' }] }; + assert.throws(() => verifyCompilerAudit(outside, options), /outside its authority/); + const missingHash = { ...report, images: [...report.images, { ...report.images[0], event: 2, kind: 'dll', path: 'C:/Windows/System32/kernel32.dll', sha256: '' }] }; + assert.throws(() => verifyCompilerAudit(missingHash, options), /unhashed/); + + // The root exits before its descendants. Its numeric PID can then reappear + // as a different compiler lifetime without changing the root's exit status. + const reused = structuredClone(report); + reused.eventCount = 30; reused.totalProcesses = 3; reused.processes[0].exitEvent = 5; + reused.processes.push( + { ...report.processes[0], instance: 2, pid: 480, startEvent: 3, exitEvent: 30, exitCode: 7 }, + { ...report.processes[0], instance: 3, startEvent: 8, exitEvent: 20, exitCode: 9 }); + reused.images.push( + { ...report.images[0], instance: 2, pid: 480, event: 3 }, + { ...report.images[0], instance: 2, pid: 480, event: 4, kind: 'dll', path: 'C:/Windows/System32/kernel32.dll' }, + { ...report.images[0], instance: 3, event: 8 }, + { ...report.images[0], instance: 3, event: 9, kind: 'dll', path: 'C:/Windows/System32/kernel32.dll' }); + assert.equal(verifyCompilerAudit(reused, options).processes[2].exitCode, 9); + for (const mutate of [ + r => { r.schema = 'aware-compiler-debug-audit/v2'; }, + r => { r.processes[2].instance = 2; }, + r => { r.processes[2].startEvent = r.images[3].event = 4; }, + r => { r.processes[2].exitEvent = null; }, + r => { r.processes[2].exitEvent = r.processes[2].startEvent; }, + r => { r.processes[2].startEvent = 31; }, + r => { r.eventCount = 31; }, + r => { r.eventCount = 29; }, + r => { r.images[4].instance = 1; }, + r => { r.images[4].pid = 480; }, + r => { r.images[4].instance = 4; }, + r => { r.images[4].event = 20; }, + r => { r.images[4].event = 21; }, + r => { r.images[4].event = 7; }, + r => { r.images[3].event = 9; r.images[4].event = 10; }, + r => { r.images[3].path = 'C:/Windows/System32/kernel32.dll'; }, + r => { r.images[4].kind = 'process'; }, + r => { r.images.push({ ...r.images[4], event: 11, kind: 'process' }); }, + r => { r.images[2].event = 5; }, + r => { r.images.reverse(); }, + r => { r.processes.reverse(); }, + r => { r.images.splice(3, 1); }, + r => { r.exitCode = 9; }, + ]) { const changed = structuredClone(reused); mutate(changed); assert.throws(() => verifyCompilerAudit(changed, options)); } + } finally { rmSync(fixture.root, { recursive: true, force: true }); } +}); diff --git a/cli/windows-compiler-fixture.mjs b/cli/windows-compiler-fixture.mjs new file mode 100644 index 000000000..0dd780f11 --- /dev/null +++ b/cli/windows-compiler-fixture.mjs @@ -0,0 +1,26 @@ +// Small authority fixtures for semantic tests; native execution uses actual compiler inputs instead. +import { mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { CLOSURE_IDS, INPUT_IDS, NONCOMPILER_TOOL_IDS } from './windows-compiler-closure.mjs'; +export const COMPILER_FIXTURE_FILES = { + 'compiler-rust-bin': ['cargo.exe', 'rustc.exe', 'rustdoc.exe', 'rustc_driver-fixture.dll', 'std-fixture.dll'], + 'compiler-rust-lib': ['rustlib/x86_64-pc-windows-msvc/lib/libstd-fixture.rlib', 'rustlib/x86_64-pc-windows-msvc/lib/libcore-fixture.rlib'], + 'compiler-msvc-bin': ['cl.exe', 'link.exe', 'lib.exe', 'c1.dll', 'c2.dll', 'msvcp140.dll', 'vcruntime140.dll'], + 'compiler-msvc-include': ['vcruntime.h'], 'compiler-msvc-lib': ['libcmt.lib', 'libvcruntime.lib'], + 'compiler-sdk-include': ['ucrt/stdio.h', 'shared/winerror.h', 'um/windows.h', 'winrt/fixture.h', 'cppwinrt/fixture.h'], + 'compiler-sdk-um-lib': ['kernel32.lib', 'user32.lib'], 'compiler-sdk-ucrt-lib': ['ucrt.lib', 'libucrt.lib'], + 'compiler-sdk-bin': ['rc.exe', 'rcdll.dll'], 'npm-cache': ['cache'], 'cargo-home': ['vendor/probe/source'], +}; +export function compilerFixture() { + const root = mkdtempSync(join(tmpdir(), 'aware-compiler-fixture-')); + const file = (name, content = name) => { const path = join(root, name); mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, content); return path; }; + const closures = Object.fromEntries(CLOSURE_IDS.map(id => { + for (const path of COMPILER_FIXTURE_FILES[id]) file(`${id}/${path}`); + return [id, join(root, id)]; + })); + return { root, file, input: { schema: 'aware-windows-repro-builder-inputs/v1', + source: { commit: 'a'.repeat(40), tree: 'b'.repeat(40), bundle: file('source.bundle') }, + inputs: Object.fromEntries(INPUT_IDS.map(id => [id, file(`inputs/${id}`)])), + tools: Object.fromEntries(NONCOMPILER_TOOL_IDS.map(id => [id, file(`tools/${id}`)])), closures } }; +} diff --git a/cli/windows-compiler-native-fixture.mjs b/cli/windows-compiler-native-fixture.mjs new file mode 100644 index 000000000..ea1792ddf --- /dev/null +++ b/cli/windows-compiler-native-fixture.mjs @@ -0,0 +1,270 @@ +// Native test support; all compiler execution uses the production closure and auditor helpers. +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { basename, dirname, join, resolve, win32 } from 'node:path'; +import { runningInputFiles, materializeClosure, verifyRustHostVersion, rejectSourceCargoConfiguration } from './build-windows-internal-repro.mjs'; +import { COMPILER_IDS, INPUT_IDS, NONCOMPILER_TOOL_IDS, inventory, copyDirectory, fileDigest, canonicalJson, discoverSystemHost, + loaderObservedWindows, systemEnvironment, auditorTempParent, exactKeys, materializeCompiler, verifyPrivateCompiler, validateCompilerLocator, runAuditedCompiler, beneath } from './windows-compiler-closure.mjs'; +import { createWindowsBuilderRecords } from './create-windows-internal-repro-inputs.mjs'; + +const windowsPath = path => win32.normalize(path).toLowerCase(); +export function verifyNativeIncludes(document, source, roots) { + exactKeys(document, ['Version', 'Data'], 'native include report'); + assert.equal(document.Version, '1.2', 'native include schema differs'); + exactKeys(document.Data, ['Source', 'ProvidedModule', 'Includes'], 'native include data'); + assert.equal(windowsPath(document.Data.Source), windowsPath(source), 'native include source differs'); + assert.equal(document.Data.ProvidedModule, ''); + const includes = document.Data.Includes; + assert.ok(Array.isArray(includes) && includes.length > 0, 'native includes are missing'); + assert.ok(includes.some(path => /windows\.h$/i.test(path)) && includes.some(path => /stdio\.h$/i.test(path)), 'real SDK and CRT include provenance'); + for (const path of includes) { + assert.ok(win32.isAbsolute(path) && roots.some(root => { + const part = win32.relative(windowsPath(root), windowsPath(path)); + return part && part !== '..' && !part.startsWith('..\\') && !win32.isAbsolute(part); + }), `unbound C header: ${path}`); + } + return includes; +} +export function verifyNativeLinkInputs(directory, compiler, object) { + const files = inventory(directory), objectName = basename(object); + for (const name of ['env.setting', 'link.rsp', objectName]) assert.ok(files.some(file => file.path === name && file.size > 0), `native link input missing: ${name}`); + const copiedObject = files.find(file => file.path === objectName); + assert.equal(copiedObject.sha256, fileDigest(object), 'native link object differs'); + const libraries = files.filter(file => /\.lib$/i.test(file.path)); + assert.ok(libraries.some(file => /^kernel32\.lib$/i.test(file.path)) && libraries.some(file => /^libcmt\.lib$/i.test(file.path)), 'real SDK and CRT library provenance'); + const declared = ['compiler-msvc-lib', 'compiler-sdk-um-lib', 'compiler-sdk-ucrt-lib'].flatMap(id => compiler.manifest.closures[id].files); + for (const file of files) { + if (['env.setting', 'link.rsp', objectName].includes(file.path)) continue; + assert.ok(/^[^/]+\.lib$/i.test(file.path) && declared.some(input => + win32.basename(input.path).toLowerCase() === file.path.toLowerCase() && input.size === file.size && input.sha256 === file.sha256), `unbound native link input: ${file.path}`); + } + return files; +} + +export function nativeBootstrapProof(root) { + const host = loaderObservedWindows(), auditScript = runningInputFiles()['compiler-audit-script']; + const original = readFileSync(auditScript, 'utf8'); + const creation = '$auditDirectory = [IO.Directory]::CreateDirectory($auditTempPath, $auditAcl)'; + const declaration = 'public static class AwareCompilerAudit {'; + assert.ok(original.includes(creation) && original.includes(declaration), 'native bootstrap proof must instrument the actual implementation'); + for (const failure of [false, true]) { + const work = join(root, `bootstrap Łódź 😀 ${failure ? 'failure' : 'success'}`); mkdirSync(work); + const captured = join(work, 'owned-temp.txt'), instrumented = join(work, 'audit.ps1'); + const encoded = Buffer.from(captured, 'utf8').toString('base64'); + let script = original.replace(creation, `${creation}\n[IO.File]::WriteAllText([Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encoded}')), $auditTempPath)`); + if (failure) script = script.replace(declaration, 'public static class AwareCompilerAudit SYNTAX_ERROR {'); + writeFileSync(instrumented, script); + const options = { locator: { tools: { powershell: host.powershell } }, + manifest: { tools: { powershell: { sha256: fileDigest(host.powershell) } }, inputs: { 'compiler-audit-script': fileDigest(instrumented) } }, + auditScript: instrumented, workRoot: work }; + if (failure) assert.throws(() => discoverSystemHost(options), error => { + const stdout = readFileSync(join(work, 'windows-host-stdout.local.bin')); + const stderr = readFileSync(join(work, 'windows-host-stderr.local.bin')); + const launch = JSON.parse(readFileSync(join(work, 'windows-host-launch.local.json'), 'utf8')); + assert.equal(launch.status, 1); assert.equal(launch.error, null); + assert.ok(stderr.length > 0); assert.equal(readFileSync(join(work, 'windows-host-command.local.log'), 'utf8'), `${stdout}${stderr}`); + assert.equal(error.message, `compiler auditor failed: 1\n${stdout}${stderr}`); + assert.equal(existsSync(join(work, 'windows-host.local.json')), false); return true; + }); + else assert.equal(windowsPath(discoverSystemHost(options).system32), windowsPath(host.system32)); + const owned = readFileSync(captured, 'utf8').replace(/^\uFEFF/, ''); + assert.equal(windowsPath(win32.dirname(owned)), windowsPath(win32.join(host.windows, 'Temp'))); + assert.match(win32.basename(owned), /^aware-compiler-audit-[0-9a-f]{32}$/); + assert.equal(existsSync(owned), false, 'bootstrap temporary directory must be removed on success and compilation failure'); + } + console.log('Native bootstrap: Unicode work paths and owned-temp cleanup passed on success and failure'); +} + +export function nativeLifecycleProof(root, sourceOverride) { + const source = sourceOverride ?? readFileSync(runningInputFiles()['compiler-audit-script'], 'utf8'); + const marker = '$utf8 = New-Object System.Text.UTF8Encoding($false)'; + assert.equal(source.split(marker).length, 2, 'unique authenticated bootstrap boundary'); + assert.equal(source.split("Add-Type -TypeDefinition @'").length, 2, 'unique production C# literal'); + assert.equal(source.split("\n'@").length, 2, 'unique production C# terminator'); + // The replay compiles the actual type; these checks also prevent an unused + // helper from satisfying the replay while Run keeps the defective old state. + for (const call of ['var lifetimes=new ProcessLifetimes(report)', 'lifetimes.RequireNew(pid)', + 'else lifetimes.RequireActive(pid)', 'lifetimes.Begin(image,process)', + 'lifetimes.Dll(Capture(pid,"dll",file,lifetimes.Handle(pid)', + 'lifetimes.End(pid,(uint)Marshal.ReadInt32(eventBuffer,16))', 'lifetimes.InitialBreakpoint(pid)']) { + assert.ok(source.includes(call), `native loop must use lifetime transition: ${call}`); + } + const work = join(root, 'lifetime replay'); mkdirSync(work); + const script = join(work, 'replay.ps1'), request = join(work, 'request.json'), output = join(work, 'report.json'); + writeFileSync(request, JSON.stringify({ output })); + writeFileSync(script, source.slice(0, source.indexOf(marker)) + ` +function Check($condition, $message) { if (!$condition) { throw $message } } +function Refused([scriptblock]$operation) { $failed = $false; try { & $operation } catch { $failed = $true }; Check $failed 'Invalid lifecycle transition was accepted' } +function NewImage([uint32]$number, [string]$kind) { + $image = New-Object AwareCompilerAudit+Image + $image.pid = $number; $image.kind = $kind; $image.path = 'C:\\private\\compiler.exe'; $image.size = 1; $image.sha256 = 'a' * 64 + return $image +} +$report = New-Object AwareCompilerAudit+Report +$state = [AwareCompilerAudit+ProcessLifetimes]::new($report) +Refused { $state.End(480, 0) } +Refused { $state.Dll((NewImage 480 'dll')) } +Refused { $state.InitialBreakpoint(480) } +$report.eventCount = 1; $first = $state.Begin((NewImage 42 'process'), [IntPtr]42) +$report.eventCount = 2; $old = $state.Begin((NewImage 480 'process'), [IntPtr]100) +Refused { $state.RequireNew(480) } +Refused { $state.Begin((NewImage 480 'process'), [IntPtr]999) } +Check ($report.processes.Count -eq 2 -and $report.images.Count -eq 2) 'Duplicate appended history' +$report.eventCount = 3 +Check ($state.InitialBreakpoint(480)) 'First breakpoint missing' +Check (!$state.InitialBreakpoint(480)) 'Repeated breakpoint was accepted' +$report.eventCount = 4; $state.Dll((NewImage 480 'dll')) +$report.eventCount = 5; [void]$state.End(480, 0) +Check ($state.Count -eq 1) 'Exited lifetime remains active' +Refused { $state.Handle(480) } +Refused { $state.RequireActive(480) } +Refused { $state.End(480, 99) } +$report.eventCount = 6; $new = $state.Begin((NewImage 480 'process'), [IntPtr]200) +Check ($new.instance -eq 3 -and $state.Handle(480) -eq [IntPtr]200) 'Reused lifetime has stale identity or handle' +Check ($state.InitialBreakpoint(480)) 'Reused lifetime inherited a breakpoint' +Check (!$state.InitialBreakpoint(480)) 'Reused lifetime accepts duplicate breakpoint' +$report.eventCount = 7; $state.Dll((NewImage 480 'dll')) +$report.eventCount = 8; [void]$state.End(480, 7) +$report.eventCount = 9; [void]$state.End(42, 0) +Check ($state.Count -eq 0 -and $state.RootExited) 'Replay did not retire all processes' +Check ($old.exitCode -eq 0 -and $old.exitEvent -eq 5 -and $new.exitCode -eq 7 -and $new.exitEvent -eq 8) 'History was overwritten' +# Reuse the root PID too; its later exit must not replace the original root status. +$report.eventCount = 10; [void]$state.Begin((NewImage 42 'process'), [IntPtr]300) +$report.eventCount = 11; [void]$state.End(42, 9) +Check ($report.exitCode -eq 0 -and $state.RootExited) 'Root identity followed a reused PID' +[IO.File]::WriteAllText($request.output, ($report | ConvertTo-Json -Depth 15), [Text.UTF8Encoding]::new($false)) +`); + const host = loaderObservedWindows(); + const result = spawnSync(host.powershell, ['-NoProfile', '-NonInteractive', '-File', script, '-RequestPath', request], + { env: systemEnvironment(host, auditorTempParent(host)), encoding: null, windowsHide: true, timeout: 30000 }); + writeFileSync(join(work, 'stdout.bin'), result.stdout ?? Buffer.alloc(0)); writeFileSync(join(work, 'stderr.bin'), result.stderr ?? Buffer.alloc(0)); + assert.ifError(result.error); assert.equal(result.status, 0, String(result.stderr)); + const report = JSON.parse(readFileSync(output, 'utf8')); + assert.deepEqual(report.processes.map(record => [record.instance, record.pid, record.startEvent, record.exitEvent, record.exitCode]), + [[1,42,1,9,0],[2,480,2,5,0],[3,480,6,8,7],[4,42,10,11,9]]); + assert.deepEqual(report.images.map(image => [image.instance, image.pid, image.event, image.kind]), + [[1,42,1,'process'],[2,480,2,'process'],[2,480,4,'dll'],[3,480,6,'process'],[3,480,7,'dll'],[4,42,10,'process']]); + console.log('Native production lifecycle replay: PID reuse, root identity, breakpoint and handle retirement passed'); +} + +function installedRoots(run) { + if (process.env.AWARE_REPRO_COMPILER_ROOTS) return JSON.parse(readFileSync(process.env.AWARE_REPRO_COMPILER_ROOTS, 'utf8')); + const rust = dirname(dirname(run('rustup', ['which', '--toolchain', '1.95.0', 'rustc']).trim())); + const msvc = process.env.VCToolsInstallDir, sdk = process.env.WindowsSdkDir, version = process.env.WindowsSDKVersion?.replace(/[\\/]+$/, ''); + assert.ok(msvc && sdk && version, 'native gate needs the VS developer environment or AWARE_REPRO_COMPILER_ROOTS'); + return { 'rust-bin': join(rust, 'bin'), 'rust-lib': join(rust, 'lib'), + 'msvc-bin': join(msvc, 'bin', 'Hostx64', 'x64'), 'msvc-include': join(msvc, 'include'), 'msvc-lib': join(msvc, 'lib', 'x64'), + 'sdk-include': join(sdk, 'Include', version), 'sdk-um-lib': join(sdk, 'Lib', version, 'um', 'x64'), + 'sdk-ucrt-lib': join(sdk, 'Lib', version, 'ucrt', 'x64'), 'sdk-bin': join(sdk, 'bin', version, 'x64') }; +} +export function prepareNativeCompiler({ base, work, source, closure, side, run }) { + console.log(`Native ${side}: copy and inventory actual compiler inputs`); + const owned = join(base, 'owned compiler inputs'), installed = installedRoots(run); + const evidence = process.env.AWARE_REPRO_TEST_EVIDENCE ? join(resolve(process.env.AWARE_REPRO_TEST_EVIDENCE), side) : join(base, 'evidence'); + mkdirSync(owned); mkdirSync(evidence, { recursive: true }); mkdirSync(join(work, 'cargo-target')); + const closures = { 'cargo-home': closure, 'npm-cache': join(owned, 'npm') }; + mkdirSync(closures['npm-cache']); writeFileSync(join(closures['npm-cache'], 'fixture'), 'unused npm fixture'); + for (const id of COMPILER_IDS) { + closures[id] = join(owned, id); + copyDirectory(installed[id.slice('compiler-'.length)], closures[id]); + } + const fixture = join(owned, 'unused-source'); writeFileSync(fixture, 'native probe authority'); + const files = runningInputFiles(), auditScript = files['compiler-audit-script']; + const { manifest, locator } = createWindowsBuilderRecords({ schema: 'aware-windows-repro-builder-inputs/v1', + source: { commit: 'a'.repeat(40), tree: 'b'.repeat(40), bundle: fixture }, + inputs: Object.fromEntries(INPUT_IDS.map(id => [id, files[id] ?? fixture])), + tools: Object.fromEntries(NONCOMPILER_TOOL_IDS.map(id => [id, id === 'powershell' ? loaderObservedWindows().powershell : process.execPath])), closures }); + const host = discoverSystemHost({ manifest, locator, auditScript, workRoot: work }); + const compiler = materializeCompiler({ manifest, locator, workRoot: work, host }); + const privateClosure = materializeClosure('cargo-home', closure, join(work, 'cargo-closure'), manifest, inventory); + writeFileSync(join(closure, 'vendor', 'path-probe', 'src', 'lib.rs'), 'changed original after copying'); + assert.equal(canonicalJson(inventory(privateClosure)), canonicalJson(manifest.closures['cargo-home'].files)); + renameSync(owned, join(base, 'hidden original compiler')); renameSync(closure, join(base, 'hidden original cargo')); + for (const path of Object.values(closures)) assert.equal(existsSync(path), false, 'original source path is unavailable'); + const audits = []; + function audit(id, args, env, label, timeout = 180000) { + const result = runAuditedCompiler({ compiler, toolPath: compiler.tools[id], args, env, label, cwd: source, + auditScript, evidenceRoot: evidence, targetRoot: join(work, 'cargo-target'), timeout }); + audits.push({ label, sha256: result.evidenceSha256, processes: result.report.processes.length }); + writeFileSync(join(evidence, `${label}.log`), result.text); return result.text; + } + function finish(record) { + verifyPrivateCompiler(compiler); + assert.equal(canonicalJson(inventory(privateClosure)), canonicalJson(manifest.closures['cargo-home'].files)); + writeFileSync(join(evidence, 'summary.json'), canonicalJson({ ...record, audits })); + } + return { compiler, privateClosure, audit, finish, evidence, source, work, locator }; +} +export function nativeVersionProof({ native, env }) { + const { compiler, audit } = native; + assert.equal(env.VCINSTALLDIR, join(compiler.root, 'msvc')); + assert.equal(env.VSCMD_ARG_TGT_ARCH, 'x64'); + assert.equal(env._NO_DEBUG_HEAP, '1'); + for (const [index, altered] of [undefined, '0', 1].entries()) { + const changed = { ...env }, label = `heap-refusal-${index}`; + if (altered === undefined) delete changed._NO_DEBUG_HEAP; else changed._NO_DEBUG_HEAP = altered; + assert.throws(() => audit('rustc', ['--version'], changed, label), /fixed _NO_DEBUG_HEAP=1/); + assert.equal(existsSync(join(native.evidence, `${label}-request.local.json`)), false); + } + assert.throws(() => audit('rustc', ['--version'], { ...env, _no_debug_heap: '0' }, 'heap-case-refusal'), /fixed _NO_DEBUG_HEAP=1/); + assert.equal(existsSync(join(native.evidence, 'heap-case-refusal-request.local.json')), false); + assert.match(audit('cargo', ['--version'], env, 'cargo-version'), /^cargo 1\.95\.0\b/); + rejectSourceCargoConfiguration(native.source, env.CARGO_HOME); + verifyRustHostVersion(audit('rustc', ['--version', '--verbose'], env, 'rust-version')); + for (const query of ['sysroot', 'target-libdir']) { + const path = audit('rustc', ['--print', query], env, `rust-${query}`).trim(); + assert.ok(existsSync(path) && beneath(path, join(compiler.root, 'rust'))); + } + assert.throws(() => audit('rustc', ['--version'], env, 'audit-deadline', 1), /compiler auditor failed/); + const incomplete = JSON.parse(readFileSync(join(native.evidence, 'audit-deadline-audit.local.json'), 'utf8')); + assert.equal(incomplete.complete, false); assert.match(incomplete.error, /deadline exceeded/); + const launch = JSON.parse(readFileSync(join(native.evidence, 'audit-deadline-launch.local.json'), 'utf8')); + assert.equal(launch.status, 1); + assert.ok(readFileSync(join(native.evidence, 'audit-deadline-stderr.local.bin')).length > 0); +} +export function nativeToolsProof({ native, env, run }) { + const { compiler, audit, evidence, source, work, locator } = native, target = join(work, 'cargo-target'); + console.log('Native compiler: mutation refusals, SDK/CRT provenance and auxiliary tools'); + for (const [index, path] of [join(compiler.roots['compiler-msvc-include'], 'vcruntime.h'), join(compiler.roots['compiler-sdk-um-lib'], 'kernel32.lib')].entries()) { + const original = readFileSync(path), label = `mutation-${index}`, output = join(target, `must-not-exist-${index}.obj`); + try { + writeFileSync(path, Buffer.concat([original, Buffer.from('mutation')])); + assert.throws(() => audit('cl', ['/c', join(source, 'native.c'), `/Fo${output}`], env, label), /private compiler changed/); + assert.equal(existsSync(join(evidence, `${label}-request.local.json`)), false); assert.equal(existsSync(output), false); + } finally { writeFileSync(path, original); } + } + assert.throws(() => { validateCompilerLocator({ ...locator, environment: { PATH: 'C:\\untrusted' } }); audit('rustc', ['--version'], env, 'hostile-locator'); }, /locator/); + assert.equal(existsSync(join(evidence, 'hostile-locator-request.local.json')), false); + const cSource = join(source, 'native.c'), object = join(target, 'native.obj'), library = join(target, 'native.lib'), executable = join(target, 'native.exe'); + const header = join(source, 'native-header.h'); + writeFileSync(header, '#include \nstatic const char *header_file = __FILE__;\nstatic const wchar_t *wide_header_file = __FILEW__;\nstatic void positive(int value) { assert(value > 0); }\n'); + writeFileSync(cSource, '#include \n#include \n#include \n#include \nint main(int argc, char **argv) { positive(argc); puts(GetCurrentProcessId() ? "private-compiler-ok" : "error"); puts(__FILE__); puts(header_file); wprintf(L"%ls\\n", wide_header_file); return 0; }\n'); + const dependencies = join(target, 'includes.json'), linkRepro = join(target, 'link-repro'); + const nativeLog = audit('cl', ['/nologo', '/utf-8', '/c', '/MT', '/showIncludes', '/sourceDependencies', dependencies, `/I${source}`, `/Fo${object}`, cSource], env, 'native-compile'); + assert.doesNotMatch(nativeLog, /D900[27]|LNK4044|option ignored/i); + const includeReport = JSON.parse(readFileSync(dependencies, 'utf8').replace(/^\uFEFF/, '')); + verifyNativeIncludes(includeReport, cSource, [compiler.roots['compiler-sdk-include'], compiler.roots['compiler-msvc-include'], source]); + writeFileSync(join(evidence, 'native-includes.json'), canonicalJson(includeReport)); + audit('lib', ['/nologo', '/Brepro', `/OUT:${library}`, win32.relative(source, object)], env, 'native-library'); + mkdirSync(linkRepro); + audit('link', ['/nologo', '/Brepro', '/VERBOSE:LIB', `/LINKREPRO:${linkRepro}`, `/OUT:${executable}`, object, 'kernel32.lib'], env, 'native-link'); + const linkInputs = verifyNativeLinkInputs(linkRepro, compiler, object); + writeFileSync(join(evidence, 'native-link-inputs.json'), canonicalJson(linkInputs)); + assert.equal(run(executable, [], { env }).trim().replaceAll('\\','/').replaceAll('\r',''), + 'private-compiler-ok\n/source/native.c\n/source/native-header.h\n/source/native-header.h'); + const unmappedObject = join(target, 'native-unmapped.obj'); + audit('cl', ['/nologo', '/utf-8', '/c', '/MT', `/I${source}`, `/Fo${unmappedObject}`, cSource], { ...env, CL: '/Brepro' }, 'native-pathmap-negative'); + const unmappedBytes = readFileSync(unmappedObject); + assert.ok(unmappedBytes.includes(Buffer.from(header)) && unmappedBytes.includes(Buffer.from(header, 'utf16le')), + 'removing the native maps must expose both narrow and wide physical header paths'); + const mappedBytes = readFileSync(object); + assert.ok(!mappedBytes.includes(Buffer.from(source)) && !mappedBytes.includes(Buffer.from(source, 'utf16le')), + 'mapped native object must not retain either physical source spelling'); + const resource = join(source, 'native.rc'), res = join(target, 'native.res'); + writeFileSync(resource, '1 RCDATA\nBEGIN\n123\nEND\n'); audit('rc', ['/nologo', `/fo${res}`, resource], env, 'native-resource'); assert.ok(existsSync(res)); + const docSource = join(source, 'docs.rs'); writeFileSync(docSource, '/// Private documentation probe.\npub fn probe() {}\n'); + audit('rustdoc', ['--crate-name', 'private_docs', docSource, '-o', join(target, 'docs')], env, 'native-rustdoc'); + assert.ok(existsSync(join(target, 'docs', 'private_docs', 'index.html'))); + return fileDigest(executable); +} diff --git a/cli/windows-vendor-repro.native.mjs b/cli/windows-vendor-repro.native.mjs new file mode 100644 index 000000000..cf10851c4 --- /dev/null +++ b/cli/windows-vendor-repro.native.mjs @@ -0,0 +1,278 @@ +// Required Windows integration: real Cargo, a dependency outside the build root, and native bytes. +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, relative, resolve } from 'node:path'; +import { createCargoBuild, assertVerboseCargoProof, verifyExtractedInputs, runningInputFiles, + normalizeBuildText, WINDOWS_LOGICAL_RUST_FLAGS } from './build-windows-internal-repro.mjs'; +import { prepareNativeCompiler, nativeVersionProof, nativeToolsProof, nativeBootstrapProof, nativeLifecycleProof } from './windows-compiler-native-fixture.mjs'; +import { beneath, loaderObservedWindows, compilerStartupPolicy, prepareNativeArchiveAdapter, nativeArchiveAdapterRecord, verifyNativeArchiveAdapter } from './windows-compiler-closure.mjs'; + +assert.equal(process.platform, 'win32', 'native repro gate requires Windows'); +assert.equal(process.arch, 'x64', 'native repro gate requires x64'); +function run(tool, args, options = {}) { + const result = spawnSync(tool, args, { encoding: 'utf8', windowsHide: true, timeout: 120000, maxBuffer: 16 * 1024 * 1024, ...options }); + assert.equal(result.error, undefined, String(result.error)); + assert.equal(result.status, 0, `${tool} ${args.join(' ')}\n${result.stdout}\n${result.stderr}`); + return `${result.stdout ?? ''}${result.stderr ?? ''}`; +} +// Canonical long form: os.tmpdir() yields an 8.3 short path on a CI runner +// (C:/Users/RUNNER~1/...), while every path the debugger reports is the expanded +// spelling, so an un-canonicalized root made the compiler's own build artifacts +// look like images loaded from outside its authority. +const root = realpathSync.native(mkdtempSync(join(tmpdir(), 'aware vendor repro '))); +// Only this fresh empty fixture changes its compression attribute. Two uncompressed +// compiler copies fit the native gate's budget and avoid repeated decompression on +// every mandatory byte check; source installations and retained evidence are untouched. +const hash = path => createHash('sha256').update(readFileSync(path)).digest('hex'); +const records = []; +let completed = false; +function physicalLeaks(directory, roots) { + return readdirSync(directory, {withFileTypes:true}).flatMap(entry => { + const file=join(directory,entry.name); + if(entry.isDirectory()) return physicalLeaks(file,roots); + if(!/\.(exe|dll|lib|rlib|rmeta)$/i.test(entry.name)) return []; + const bytes=readFileSync(file), texts=[bytes.toString('latin1'),bytes.toString('utf8'),bytes.toString('utf16le'),bytes.subarray(1).toString('utf16le')].map(x=>x.toLowerCase()); + return roots.some(root=>[root,root.replaceAll('\\','/'),root.replaceAll('\\','\\\\')].some(p=>texts.some(text=>text.includes(p.toLowerCase())))) ? [file] : []; + }); +} +function compiledInventory(directory, root=directory) { + return readdirSync(directory,{withFileTypes:true}).sort((a,b)=>a.name.localeCompare(b.name)).flatMap(entry=>{ + const file=join(directory,entry.name);if(entry.isDirectory())return compiledInventory(file,root); + if(!/\.(exe|dll|lib|rlib|rmeta)$/i.test(entry.name))return []; + const bytes=readFileSync(file);return [{path:relative(root,file).replaceAll('\\','/'),size:bytes.length,sha256:hash(file)}]; + }); +} +function archiveMembers(file) { + const bytes=readFileSync(file);assert.equal(bytes.subarray(0,8).toString(),'!\n');let pos=8,longnames;const rows=[]; + while(pos=0&&pos+60+size<=bytes.length); + const data=bytes.subarray(pos+60,pos+60+size);if(name==='//')longnames=data;else if(name!=='/')rows.push({name,data});pos+=60+size+(size%2); + } + return rows.map(row=>{let name=row.name;if(/^\/\d+$/.test(name)){assert.ok(longnames);const offset=Number(name.slice(1)),end=longnames.indexOf(0,offset);assert.ok(end>=offset);name=longnames.subarray(offset,end).toString();} + return {name,size:row.data.length,sha256:createHash('sha256').update(row.data).digest('hex')};}); +} +try { + run(join(loaderObservedWindows().system32, 'compact.exe'), ['/U', root]); + nativeBootstrapProof(root); + nativeLifecycleProof(root); + for (const side of ['a', 'b']) { + const base = join(root, `builder ${side} Łódź 😀 with a supported long source location`), work = join(base, 'work'), source = join(work, 'source'), crateRoot = join(source, 'cli'); + const closure = join(base, 'sealed cache'), vendor = join(closure, 'vendor'), dependency = join(vendor, 'path-probe'), macro = join(vendor,'path-macro'); + const cargoHome = join(work, 'cargo-home'), tempRoot = join(work, 'temp'); + for (const dir of [join(crateRoot, 'src'), join(dependency, 'src'), join(macro,'src'), cargoHome, tempRoot]) mkdirSync(dir, { recursive: true }); + writeFileSync(join(dependency, 'Cargo.toml'), '[package]\nname="path-probe"\nversion="0.1.0"\nedition="2021"\n'); + writeFileSync(join(dependency, 'src', 'lib.rs'), 'pub fn origin() -> &\'static str { file!() }\n'); + writeFileSync(join(dependency, '.cargo-checksum.json'), JSON.stringify({ package: '0'.repeat(64), files: { + 'Cargo.toml': hash(join(dependency, 'Cargo.toml')), 'src/lib.rs': hash(join(dependency, 'src', 'lib.rs')), + } })); + writeFileSync(join(macro,'Cargo.toml'),'[package]\nname="path-macro"\nversion="0.1.0"\nedition="2021"\n[lib]\nproc-macro=true\n'); + writeFileSync(join(macro,'src','lib.rs'),'extern crate proc_macro;\n#[proc_macro]\npub fn origin(_: proc_macro::TokenStream) -> proc_macro::TokenStream { format!("{:?}", file!()).parse().unwrap() }\n'); + writeFileSync(join(macro,'.cargo-checksum.json'),JSON.stringify({package:'1'.repeat(64),files:{'Cargo.toml':hash(join(macro,'Cargo.toml')),'src/lib.rs':hash(join(macro,'src','lib.rs'))}})); + writeFileSync(join(crateRoot, 'Cargo.toml'), '[package]\nname="vendor-repro-probe"\nversion="0.1.0"\nedition="2021"\n[dependencies]\npath-probe="=0.1.0"\npath-macro="=0.1.0"\n[build-dependencies]\npath-probe="=0.1.0"\n[profile.release]\ndebug=0\n'); + writeFileSync(join(crateRoot, 'src', 'main.rs'), 'extern "C" { fn native_one() -> i32; fn native_two() -> i32; }\nfn main() { println!("{}", path_probe::origin()); println!("{}", path_macro::origin!()); /* SAFETY: fixture functions are linked from the two audited C objects, take no arguments and return integers. */ println!("{}", unsafe { native_one()+native_two() }); }\n'); + const native = prepareNativeCompiler({ base, work, source, closure, side, run }); + const { compiler, privateClosure, audit } = native; + symlinkSync(crateRoot, join(crateRoot, 'archive-junction'), 'junction'); + const denied = compilerStartupPolicy(compiler).deniedImage; + // Always exercise inherited heap configuration, even when optional telemetry is absent. + // The negative child loses the flag AFTER the valid root request, so it tests + // descendant inheritance rather than merely tripping the launcher's preflight. + writeFileSync(join(crateRoot, 'build.rs'), ` +#[link(name = "kernel32")] +extern "system" { fn IsDebuggerPresent() -> i32; } +fn main() { + assert_eq!(std::env::var("_NO_DEBUG_HEAP").as_deref(), Ok("1"), "AWARE_HEAP_SETTING_MISSING:_NO_DEBUG_HEAP"); + // SAFETY: documented zero-argument Win32 query; no pointers or borrowed memory. + assert_ne!(unsafe { IsDebuggerPresent() }, 0, "auditor must remain attached"); + if std::env::args().nth(1).as_deref() == Some("heap-child") { + println!("AWARE_HEAP_CHILD_OK"); + return; + } + println!("cargo:warning=AWARE_HOST_PATH:{}", path_probe::origin()); + let executable = std::env::current_exe().unwrap(); + let mut positive = std::process::Command::new(&executable); + positive.arg("heap-child").stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()); + let child = positive.spawn().unwrap(); let positive_pid = child.id(); + let good = child.wait_with_output().unwrap(); + assert_eq!(good.status.code(), Some(0)); + assert_eq!(String::from_utf8_lossy(&good.stdout).trim(), "AWARE_HEAP_CHILD_OK"); + let mut negative = std::process::Command::new(&executable); + negative.arg("heap-child").env_remove("_NO_DEBUG_HEAP").stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()); + let child = negative.spawn().unwrap(); let negative_pid = child.id(); + let bad = child.wait_with_output().unwrap(); + assert_eq!(bad.status.code(), Some(101), "missing inherited flag must panic"); + assert!(String::from_utf8_lossy(&bad.stderr).contains("AWARE_HEAP_SETTING_MISSING:_NO_DEBUG_HEAP")); + assert!(!String::from_utf8_lossy(&bad.stdout).contains("AWARE_HEAP_CHILD_OK")); + ${denied ? `let telemetry = std::path::PathBuf::from(std::env::var("VCINSTALLDIR").unwrap()).join("bin/vctip.exe"); + let status = std::process::Command::new(telemetry).status().expect("private telemetry child"); + assert_eq!(status.code().unwrap() as u32, ${denied.exitCode}u32);` : ''} + let out = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let mut objects = Vec::new(); + for (name, value) in [("one",1),("two",2)] { + let dir=out.join(name); std::fs::create_dir_all(&dir).unwrap(); + let source=dir.join("probe.c"); let object=dir.join("probe.o"); + std::fs::write(&source, format!("int native_{}(void) {{ return {}; }}", name,value)).unwrap(); + let result=std::process::Command::new(std::env::var("CC").unwrap()).args(["/nologo","/MT","/O2","/c"]).arg(format!("/Fo{}",object.display())).arg(source).output().unwrap(); + assert!(result.status.success(),"native compiler: {:?}",result); + objects.push(object); + } + let library=out.join("nativeprobe.lib"); + for (index,object) in objects.iter().enumerate() { + let mut command=std::process::Command::new(std::env::var("AR").unwrap()); + command.arg(format!("-out:{}",library.display())).arg("-nologo"); + if index>0 { command.arg(&library); } + let result=command.arg(object).output().unwrap(); assert!(result.status.success(),"native librarian: {:?}",result); + } + let before=std::fs::read(&library).unwrap(); + for alias in ["COM¹", "COM²", "COM³", "LPT¹", "LPT²", "LPT³", "CONIN$", "CONOUT$", "CLOCK$", "COM1 "] { + for args in [vec![format!("/OUT:{}",out.join(format!("{}.lib",alias)).display()),objects[0].display().to_string()], + vec![format!("/OUT:{}",library.display()),out.join(format!("{}.obj",alias)).display().to_string()]] { + let refused=std::process::Command::new(std::env::var("AR").unwrap()).args(args).output().unwrap(); + assert_eq!(refused.status.code(),Some(2)); + assert!(String::from_utf8_lossy(&refused.stderr).contains("reserved device component")); + assert_eq!(std::fs::read(&library).unwrap(),before,"device refusal must not mutate the archive"); + } + } + let output_arg=format!("/OUT:{}",library.display()); + let object_arg=objects[0].display().to_string(); + let other=out.join("other.lib"); std::fs::copy(&library,&other).unwrap(); + let cwd=std::env::current_dir().unwrap(); + let response=cwd.join("@payload.obj"); let option=cwd.join("-payload.obj"); + let escaped=cwd.join("must-not-escape.lib"); + std::fs::write(&response,format!("/OUT:{} {}",escaped.display(),objects[0].display())).unwrap(); + std::fs::copy(&objects[0],&option).unwrap(); + for args in [vec![output_arg.clone(),"@inputs.rsp".into()], + vec![output_arg.clone(),response.display().to_string()], + vec![output_arg.clone(),option.display().to_string()], + vec![output_arg.clone(),cwd.join("archive-junction/-payload.obj").display().to_string()], + vec![output_arg.clone(),"/UNKNOWN".into(),object_arg.clone()], + vec![output_arg.clone(),"C:drive.obj".into()], + vec![output_arg.clone(),output_arg.clone(),object_arg.clone()], + vec![output_arg.clone(),other.display().to_string(),object_arg.clone()], + vec![format!("/OUT:{}/../../escape.lib",out.display()),object_arg.clone()]] { + let reparse_case=args.iter().any(|arg|arg.contains("archive-junction")); + if reparse_case { assert!(cwd.join("archive-junction/-payload.obj").is_file()); } + let refused=std::process::Command::new(std::env::var("AR").unwrap()).args(args).output().unwrap(); + assert_eq!(refused.status.code(),Some(2)); + assert!(String::from_utf8_lossy(&refused.stderr).contains("AWARE_NATIVE_ARCHIVE_REFUSED")); + if reparse_case { assert!(String::from_utf8_lossy(&refused.stderr).contains("reparse path is forbidden")); } + assert_eq!(std::fs::read(&library).unwrap(),before,"refusal must not mutate the archive"); + assert!(!escaped.exists(),"converted filename must never inject librarian options"); + } + println!("cargo:rustc-link-search=native={}",out.display()); println!("cargo:rustc-link-lib=static=nativeprobe"); + println!("cargo:warning=AWARE_HEAP_PROOF parent={} positive={} negative={}", std::process::id(), positive_pid, negative_pid); +} +`); + const { cargoVendor, env, args } = createCargoBuild({ compiler, workRoot: work, sourceRoot: source, cargoHome, cargoClosure: privateClosure, tempRoot }); + nativeVersionProof({ native, env }); + const adapter=prepareNativeArchiveAdapter(work), flags=env.CARGO_ENCODED_RUSTFLAGS.split('\x1f'); + audit('rustc',[adapter.source,'--crate-name','aware_native_archive_adapter','--edition=2021','-C','opt-level=2','-C','debuginfo=0','-C',`linker=${compiler.tools.link}`,...flags,'-o',adapter.executable],env,'native-archive-adapter-build'); + const adapterRecord=nativeArchiveAdapterRecord(base); + const config = args.flatMap((arg, index) => arg === '--config' ? [arg, args[index + 1]] : []); + audit('cargo', ['generate-lockfile', '--manifest-path', join(crateRoot, 'Cargo.toml'), '--offline', ...config], env, 'cargo-lock'); + const verbose = audit('cargo', args, env, 'cargo-build'); + assert.ok(verbose.includes('AWARE_HOST_PATH:'), 'real host build dependency receives its specific remap'); + const heapProof = /cargo:warning=AWARE_HEAP_PROOF parent=(\d+) positive=(\d+) negative=(\d+)/.exec(verbose); + assert.ok(heapProof, 'audited build-script parent authenticated both inherited-heap outcomes'); + const heapAudit = JSON.parse(readFileSync(join(native.evidence, 'cargo-build-audit.local.json'), 'utf8')); + verifyNativeArchiveAdapter(adapterRecord,heapAudit,base); + const parentCandidates = heapAudit.processes.filter(process => process.pid === Number(heapProof[1]) + && /[\\/]vendor-repro-probe-[^\\/]+[\\/]build-script-build\.exe$/i.test(process.path)); + assert.equal(parentCandidates.length, 1, 'one exact build-script parent lifetime'); + const parent = parentCandidates[0]; assert.equal(parent.exitCode, 0); + for (const [pid, exitCode] of [[Number(heapProof[2]), 0], [Number(heapProof[3]), 101]]) { + const children = heapAudit.processes.filter(process => process.pid === pid && process.path === parent.path && process.exitCode === exitCode + && process.startEvent > parent.startEvent && process.exitEvent < parent.exitEvent); + assert.equal(children.length, 1, 'one exact nested child lifetime within the observed parent'); + assert.equal(children[0].exitCode, exitCode, 'audited child exit matches its authenticated outcome'); + const images = heapAudit.images.filter(image => image.instance === children[0].instance && image.kind === 'process'); + assert.equal(images.length, 1); assert.equal(images[0].path, parent.path); + assert.equal(images[0].sha256, hash(parent.path), 'nested child image binds the actual build script'); + } + if (denied) { + const proof = JSON.parse(readFileSync(join(native.evidence, 'cargo-build-audit.local.json'), 'utf8')); + assert.ok(proof.processes.some(process => process.action === 'blocked-telemetry' && process.exitCode === denied.exitCode), 'real private telemetry descendant was denied and its exit observed'); + } + assert.equal(normalizeBuildText(flags.join(' '), [[work, ''], [source, ''], + [cargoHome, ''], [cargoVendor, '']]), WINDOWS_LOGICAL_RUST_FLAGS, + 'the receipt records the complete logical compiler arguments'); + assertVerboseCargoProof(`${args.join(' ')}\n${verbose}`, flags, compiler.tools.rustc); + const executable = join(work, 'cargo-target', 'release', 'vendor-repro-probe.exe'); + const origin = run(executable, [], { env }).trim().replaceAll('\\', '/'); + assert.equal(origin.replaceAll('\r',''), '/path-probe/src/lib.rs\n/path-macro/src/lib.rs\n3'); + const goodHash = hash(executable); + assert.deepEqual(physicalLeaks(join(work,'cargo-target'),[base,vendor,compiler.root]),[], 'host dependencies, proc macros, native archives and final images contain no physical roots'); + const compiled=compiledInventory(join(work,'cargo-target')); + const libraryRecord=compiled.find(record=>record.path.endsWith('/nativeprobe.lib'));assert.ok(libraryRecord); + const library=join(work,'cargo-target',...libraryRecord.path.split('/')),members=archiveMembers(library); + assert.deepEqual(members.map(member=>member.sha256).sort(),['one','two'].map(name=>hash(join(dirname(library),name,'probe.o'))).sort(), 'archive preserves both same-basename object payloads exactly'); + assert.ok(members.every(member=>!member.name.includes(base)&&member.name.includes('cargo-target')), 'archive names retain relative directories, not physical roots'); + + // Exercise both argument spellings directly, in addition to Cargo's native dependency invocation. + for (const [index, spelling] of [join(cargoVendor, 'path-probe', 'src', 'lib.rs'), join(cargoVendor, 'path-probe', 'src', 'lib.rs').replaceAll('\\', '/')].entries()) { + const probeSource = join(source, 'spelling.rs'); + writeFileSync(probeSource, `#[path=${JSON.stringify(spelling)}] mod dependency;\nfn main() {println!("{}", dependency::origin());}\n`); + const output = join(work, 'spelling.exe'); + audit('rustc', [probeSource, '--crate-name', 'spelling_probe', '-O', '-C', 'debuginfo=0', '-C', `linker=${compiler.tools.link}`, ...flags, '-o', output], env, `rust-spelling-${index}`); + assert.equal(run(output, [], { env }).trim().replaceAll('\\', '/'), '/path-probe/src/lib.rs'); + } + + const badFlags = flags.filter(flag => !flag.endsWith('=')); + assert.equal(flags.length - badFlags.length, 2, 'mutation removes BOTH Windows vendor spellings'); + const badEnv = { ...env, CARGO_ENCODED_RUSTFLAGS: badFlags.join('\x1f'), CARGO_TARGET_DIR: join(work, 'cargo-target', 'mutated-target') }; + audit('cargo', args, badEnv, 'cargo-red-control'); + const badExecutable = join(work, 'cargo-target', 'mutated-target', 'release', 'vendor-repro-probe.exe'); + const badOrigin = run(badExecutable, [], { env: badEnv }).trim().replaceAll('\\', '/'); + assert.equal(badOrigin.replaceAll('\r',''), '/cargo-closure/vendor/path-probe/src/lib.rs\n/cargo-closure/vendor/path-macro/src/lib.rs\n3', 'mutation exposes only the broad work remap'); + assert.notEqual(hash(badExecutable), goodHash, 'mutation changes the executable'); + const cHash = nativeToolsProof({ native, env, run }); + const hostEnv={...env,CARGO_TARGET_DIR:join(work,'cargo-target','host-negative')}; + const hostArgs=[...args,'--target','x86_64-pc-windows-msvc']; + const hostLog=audit('cargo',hostArgs,hostEnv,'cargo-host-red-control'); + assert.throws(()=>assertVerboseCargoProof(hostArgs.join(' ')+'\n'+hostLog,flags,compiler.tools.rustc),/omitted/); + assert.ok(physicalLeaks(join(work,'cargo-target','host-negative'),[base]).length>0,'explicit target mutation exposes real host paths'); + records.push({ side, origin, goodHash, badOrigin, badHash: hash(badExecutable), cHash, adapterHash:hash(adapter.executable),compiled }); + native.finish(records.at(-1)); + assert.ok(beneath(base, root) && resolve(base) !== resolve(root)); + } + assert.equal(records[0].goodHash, records[1].goodHash, 'independent Cargo builds are byte-identical'); + assert.equal(records[0].cHash, records[1].cHash, 'independent native C executables are byte-identical'); + assert.equal(records[0].adapterHash, records[1].adapterHash, 'independent source-bound archive adapters are byte-identical'); + assert.deepEqual(records[0].compiled,records[1].compiled,'complete compiled host/native artifact inventories match across roots'); + // The private vendor now lives under work: removing its specific remaps changes both + // executables to the same wrong logical path, rather than exposing the original cache root. + for (const record of records) assert.notEqual(record.badHash, record.goodHash); + + // Real Git bundle materialization must not let an old runner build a different source script. + const repo = join(root, 'source authority'), extracted = join(root, 'extracted source'); + mkdirSync(join(repo, 'cli'), { recursive: true }); mkdirSync(join(repo, 'cli-connection-reader')); + const oldRoot = join(root, 'old source'); mkdirSync(join(oldRoot, 'cli'), { recursive: true }); mkdirSync(join(oldRoot, 'cli-connection-reader')); + const script = join(repo, 'cli', 'build-windows-internal-repro.mjs'), running = join(oldRoot, 'cli', 'build-windows-internal-repro.mjs'); + for (const builder of [script, running]) for (const [id, path] of Object.entries(runningInputFiles(builder))) writeFileSync(path, id); + writeFileSync(join(repo, 'cli', 'Cargo.lock'), 'lock'); writeFileSync(join(repo, 'cli-connection-reader', 'package-lock.json'), '{}'); + const git = 'git'; + run(git, ['init', repo]); run(git, ['-C', repo, 'add', '.']); + const commitArgs = ['-c', 'user.name=Repro fixture', '-c', 'user.email=repro@example.invalid', '-c', 'commit.gpgsign=false', '-c', 'core.hooksPath=NUL', 'commit', '-qm']; + run(git, ['-C', repo, ...commitArgs, 'old fixture']); + const inputs = { 'aware-cargo-lock': hash(join(repo, 'cli', 'Cargo.lock')), 'reader-package-lock': hash(join(repo, 'cli-connection-reader', 'package-lock.json')), + ...Object.fromEntries(Object.entries(runningInputFiles(script)).map(([id, path]) => [id, hash(path)])) }; + writeFileSync(script, 'new builder'); run(git, ['-C', repo, 'add', '.']); run(git, ['-C', repo, ...commitArgs, 'new fixture']); + const bundle = join(root, 'source.bundle'); run(git, ['-C', repo, 'bundle', 'create', bundle, 'HEAD']); run(git, ['clone', bundle, extracted]); + assert.throws(() => verifyExtractedInputs(extracted, inputs, running), /extracted source/); + inputs['builder-script'] = hash(join(extracted, 'cli', 'build-windows-internal-repro.mjs')); + assert.throws(() => verifyExtractedInputs(extracted, inputs, running), /extracted source/); + console.log(`Windows Cargo vendor-path repro passed: 2 byte-identical executables, both path spellings, spaced paths, two red mutations, old-runner/new-bundle refusal. ${records[0].goodHash}`); + completed = true; +} finally { + // This test owns the unique temporary directory it created above. Compare + // canonical against canonical: `root` is the expanded spelling, so an + // un-canonicalized tmpdir() (C:/Users/RUNNER~1/... on CI) would make the + // directory this test just created look like it sits outside its own parent. + const temporaryParent = realpathSync.native(tmpdir()); + if (!beneath(root, temporaryParent) || resolve(root) === resolve(temporaryParent)) throw new Error('test cleanup escaped its temporary parent'); + if (completed) rmSync(root, { recursive: true, force: true }); + else console.error(`Native failure evidence retained: ${root}`); +} diff --git a/docs/superpowers/specs/2026-08-30-reproducible-windows-runtime-plan.md b/docs/superpowers/specs/2026-08-30-reproducible-windows-runtime-plan.md new file mode 100644 index 000000000..8ba524032 --- /dev/null +++ b/docs/superpowers/specs/2026-08-30-reproducible-windows-runtime-plan.md @@ -0,0 +1,84 @@ +# Plan: reproducible private Windows runtime and isolated RVT `.2` test media +_Round 4 revision after adversarial review_ + +## Goal and claim boundary + +Produce a path-independent, byte-reproducible AWARE Windows runtime from retained immutable source inputs, stage it canonically into FloLess, and use it to build and UAT private `0.136.0-rvt.2` without touching the ordinary local production installation, process, port, or state roots. Production-backed service UAT is a separate, explicitly bounded authority gate in section 7; no general “production untouched” claim is made for those authorized requests. The proof must use two freshly provisioned Windows builders with no shared mutable caches or checkout state. Because no independently controlled signing key has been allocated, `.2` is explicitly **unsigned, unauthenticated internal test media**: it may prove source-to-byte reproducibility and runtime behavior, but it is not releasable, install-authorized production media, or evidence of publisher identity. + +## 1. Anchor pre-change recovery inputs before editing + +- Verify the retained `.1` runtime manifest against the already approved SHA-256 `c94d20869d2e09b0866b2a7e2605ed9532a8dac25e08c1b6a239e9beffe63317`, then verify every payload record. Verify the retained accepted containers against these fixed anchors: full NUPKG `505f584911d8b21b0d25db2717d3b8a511dbf8bed502be976f8bcbae3f8c95d5` (120,071,678 bytes), portable ZIP `5a39027ab3daa4d1510d97c4616c44c07b4aea395cb6d7231c1491b4206d446b` (120,027,063 bytes), and Setup EXE `2f69ebd96c4866651b280036a1754afddcd2fa4092bf94c248ec34b50002479b` (124,577,278 bytes). Stop if any anchor differs. +- Create **pre-change recovery bundles** for the current exact AWARE and FloLess tips before editing. Verify each self-contained Git bundle and record its SHA-256/size plus contained commit/tree IDs. These preserve the starting point but are not the later build inputs. +- Record and hash the complete builder-input set: Windows image identity; Rust/rustc/cargo; MSVC `cl.exe`/`link.exe`/`lib.exe`; Windows SDK tools and linked libraries; Node `24.14.0` base executable; npm; all esbuild/postject/rcedit/reader native and WASM bytes; AWARE `cli/Cargo.lock`; reader `cli-connection-reader/package-lock.json`; and FloLess `server/package-lock.json`. Reject an input that is absent, floating, or digest-mismatched. +- Materialize an initial closed offline dependency archive for recovery/tooling only: `cargo vendor` output for every locked registry/git crate plus Cargo source replacement config; npm tarballs/platform packages for every current reader and FloLess lock record (including `fflate`, `web-ifc`, esbuild, postject, rcedit, and their native/WASM payloads); and the exact allowed lifecycle scripts. Hash every archive entry and provenance record. This pre-change archive is never silently reused for a final build. +- After each final or post-merge AWARE/FloLess build-input bundle is created, derive a **new generation-specific offline closure from those bundles' exact three lockfiles**, verify every lock record is present and no extra executable input is admitted, and bind the closure digest/inventory to that generation's builder manifest. Both isolated builds use that closure with network disabled and fail if a lifecycle script attempts a download or produces an undeclared byte. A lockfile change always requires a new closure and builder manifest before build. +- Preserve the two prior failed detached-build inventories and add a read-only PE/SEA difference diagnostic. Evidence must contain no secret, username, absolute checkout path, or ambient environment dump. + +## 2. Add a dedicated fail-closed AWARE internal build entrypoint + +- Leave the public release workflow and its tag-version/Google-secret behavior unchanged. Add a separate Windows-x64 internal reproducibility wrapper that consumes only a verified AWARE Git bundle plus a closed builder manifest. +- The wrapper starts from an empty output root and an explicit environment allowlist. It clears or rejects ambient build authority including `RUSTFLAGS`, `CARGO_ENCODED_RUSTFLAGS`, `CC`, `CFLAGS`, `CL`, `LINK`, `LIB`, `INCLUDE`, `NODE_OPTIONS`, `npm_config_*`, `ESBUILD_BINARY_PATH`, credentials, and compile-time authority. It then sets only reviewed absolute tool paths, vendored source configuration, offline mode, and deterministic flags; any network socket/use is a failure. +- Build Rust with the committed `1.95.0` toolchain and `cargo build --release --locked`. Apply `/Brepro` to rustc's final MSVC link and to native `cc`/MSVC compilation such as `ring`; do this in the wrapper/build configuration, not by modifying portable debug or non-Windows builds. +- Capture the real verbose Cargo/native compiler execution and behaviorally assert the locked invocation, exact tools, target/profile, and reproducible flags. Tests must fail for a poisoned environment, wrong tool digest, wrong target, omitted lock, omitted `/Brepro`, or a dead/uninvoked guard. + +## 3. Make the connection-reader SEA deterministic + +- Change `cli-connection-reader/build.mjs` so `sea-config.json` uses fixed relative `main` and `output` names and every Node/postject step runs with the immutable output directory as `cwd`; no source or output root may enter the bundle, SEA blob, or executable. +- Fail closed unless Node is exactly `24.14.0`, Windows x64, and the base `node.exe` digest matches the builder manifest. Use `npm ci --offline` from the verified lock and retained tarball cache, with only the closed lifecycle-script allowlist, and invoke absolute digest-checked Node/npm/esbuild/postject inputs. +- Emit a canonical build receipt in the immutable output directory. It binds the **final build-input** bundle/commit/tree, closed toolchain manifest digest, normalized logical tool IDs, normalized root tokens, the closed set of admitted non-secret settings, package-lock/Cargo.lock digests, normalized command templates, and bundle/blob/EXE/WASM hashes. It contains no real root, time, username, nonce, or secret. Retain raw per-builder verbose commands/environments separately as non-reproducible local evidence; compare their normalized forms but never require their bytes to match or ship them. +- Add behavioral/mutation tests for wrong Node/tool/package bytes, absolute SEA paths, wrong cwd, ambient-variable poisoning, receipt closure, and post-build tampering. + +## 4. Prove reproducibility on two isolated builders + +- Provision two fresh Windows Sandbox/VM instances independently from the recorded image, give each only identical hash-verified source/toolchain inputs, and use separate local Git object stores, Cargo/npm registries, temp roots, output roots, caches, and user profiles. Deliberately choose different path identities and lengths for all those roots. A second worktree on the same host does not count. +- Run the real wrapper in both builders. Require byte identity for `aware.exe`, reader bundle, SEA blob, reader EXE, WASM, and canonical receipt, plus matching complete ordered inventories. +- Scan every compared artifact for both builders' checkout/target/temp/cache/output/profile paths in UTF-8, UTF-16LE, slash-normalized, JSON-escaped, URI, and case-normalized forms; any occurrence fails the proof even when the two digests happen to match. +- Run the relevant Rust tests, clippy, CLI smoke/version checks, connection-reader tests, packaged harness, and receipt mutations in both. If either clean builder cannot be provisioned or any byte differs, stop before FloLess staging; do not allowlist residual differences. +- The result proves reproducibility under the recorded toolchain, not resistance to a compromised common image or publisher identity. That limitation remains explicit until an independent signing/attestation authority is allocated. + +## 5. Stage canonical AWARE bytes into FloLess atomically + +- Advance the AWARE pin only after the reproducibility commit exists. Materialize the exact closed reader-agent subtree and `cli-npm/package.json` from Git blob objects in the verified AWARE bundle; never recursively copy a checkout. +- Before materialization reject non-blobs, symlinks/submodules, LFS pointers, unexpected attributes/types, non-UTF-8 names, case-fold collisions, reserved NTFS device names, ADS colons, and names normalized by trailing dots/spaces. Reject missing/extra tracked payload records and untracked/ignored payload candidates. +- Introduce runtime manifest schema v2 atomically because the receipt is shipped. Update the build verifier, installed verifier, layout, generation installer, generation marker, `distribution.json`, startup diagnostics, UAT evidence, and mutation tests to bind the receipt/build ID and its exact digest. `.2` staging, installation, and boot accept **only v2**. Put v1 parsing in a separate read-only transition-audit utility that is unreachable from staging/boot; the retained `.1` executable remains the only runtime that handles its own v1 media. +- Add a separate hermetic FloLess private-build wrapper. It consumes only the verified final FloLess bundle, extracts to an empty root, performs `npm ci --offline` from the retained `server/package-lock.json` cache with the closed lifecycle allowlist, verifies the complete installed dependency/native-file inventory, and invokes only absolute digest-bound Node/esbuild/postject/rcedit tools. Ambient `process.execPath`, PATH tools, ignored `server/node_modules`, and an ordinary working tree are not accepted build inputs. +- Before any staging/output mutation, the outer wrapper acquires the fixed machine-wide mutex `Global\\FlolessRvtStaging.Package.` with an explicit restrictive mutex ACL. Acquisition is non-waiting/fail-closed; inability to create the global mutex is a hard stop, never a fallback to a local lock. +- While holding the mutex, create `%ProgramData%\\FlolessRvtStaging\\version-seals\\.json` with create-new semantics and a restrictive ACL denying later delete/overwrite to the build identity. The already-open handle retains its initially granted write right and begins with a flushed permanent `reserved` record binding candidate version, build ID, source-bundle and builder-manifest digests. Any existing, malformed, reserved, or completed seal refuses the build forever; there is no stale-seal repair/deletion path. A failure after reservation burns that version and requires incrementing the private revision. Retain the handle with no write/delete sharing throughout the build; after all hashes exist, truncate and durably rewrite through that same handle to the canonical `completed` record, then flush before releasing it. A crash/torn rewrite remains malformed and therefore permanently blocking. The wrapper never removes the seal. Tests cover concurrent attempts, sequential rebuilds, crash-after-reservation, torn completion, altered sources, and a denied-delete mutation. +- The same outer wrapper process retains the mutex and seal handles continuously through source/output verification, staging, finalization, pre-pack hashing, Velopack completion, and final package hashing. A separate ephemeral owner diagnostic may be removed only by its creating build after the permanent seal is completed. +- Produce into a uniquely owned directory, make verified outputs immutable, and copy/hash through the same opened handles; fail if identity/size changes. No mutable ignored `target`, `dist`, or `node_modules` path is accepted as an input. + +## 6. Close the `.1` transition and version boundary + +- Preserve `.1` unchanged. Compare its anchored ordered runtime inventory with the first reproducible candidate under a closed policy: exact Git-blob LF bytes may replace checkout-normalized CRLF text; AWARE/reader binaries and the new v2 receipt/provenance may change; no other path or semantic content may change. +- For the specific tracked text paths listed by the transition utility, permit exactly one transform: bytes must be valid UTF-8 without BOM or lone CR, and replacing every CRLF with LF must produce the exact candidate Git blob including its final-newline state. Reject every other byte, encoding, ordering, scalar-style, duplicate-key, or content difference; do not parse Markdown/YAML/JSON to declare semantic equivalence. Binary differences require the two-builder identity proof, correct versions, matching receipts, passing runtime tests, and existing provider/runtime verification. +- Generate `prepack-payload-manifest.json` last over every staged payload byte and provenance field **except that manifest itself**. Package the manifest beside the payload. It includes and therefore binds `distribution.json`; `distribution.json` does not contain the pre-pack manifest digest. The external UAT evidence envelope records the pre-pack manifest digest and, only after packaging, the final package/container hashes. Neither external hash is written back into the package, eliminating self-reference. Use the pre-pack manifest digest—not Velopack timestamps or ordering—as version-decision identity. +- Fix FloLess private SEA absolute-path leakage. Retain the exact Velopack 1.2.0 NuGet package, dotnet host/runtime, and complete resolved 449-file tool closure; bind every file in the builder manifest and recreate it offline in a fixed private tool root. Invoke the verified private `dotnet.exe exec` directly with explicit chosen `vpk.dll`, `.runtimeconfig.json`, and `.deps.json`—never the framework-dependent global apphost. Clear/reject all ambient `DOTNET_*` and `COREHOST_*`, then set only the private root, disable multilevel lookup and roll-forward, PATH resolution, startup hooks, additional deps, shared stores, and servicing. Capture loaded modules and fail unless every non-OS module resolves beneath the verified private dotnet/Velopack root and matches its manifest digest. Do not claim final unsigned package reproducibility unless a two-package test proves it. +- Once `.2` is produced, retain its package hash and pre-pack manifest hash and never rebuild different `.2` media. Any later pre-pack payload/provenance change requires `.3`, even if the functional change appears harmless. + +## 7. Commit, verify, install, and UAT in immutable units + +- Commit the AWARE wrapper/SEA/reproducibility unit first on `codex/xeorvt-aware-rvt-reader`, without amend or push. **After that commit**, create and verify a new self-contained final AWARE build-input bundle containing it. Build that exact bundle on both isolated builders and retain both receipts/inventories. +- Commit FloLess schema-v2/canonical-staging/packaging changes separately on `codex/xeorvt-reference-model`, preserving verifier commit `2925325e516373b0bd3fd14c82c8fd6dee77c7ec`. **After that commit**, create and verify the final FloLess build-input bundle used for packaging. Compare the exact sorted `server/**/*.test.ts` inventory with the pre-change recovery bundle and a committed, explicitly reviewed add/delete/rename delta; an unlisted disappearance fails. Run the complete dynamically discovered suite, record file/test/pass/fail/skip/todo/cancel counts from TAP, and reject every skip/todo/cancel outside the committed existing allowlist. Do not claim an assertion count without runtime assertion instrumentation. Also run typecheck, boundary, all wired guards, staging mutations, transition audit, and exact-commit build. +- Build private `.2`, label it visibly and in metadata as unsigned internal test media, verify package layout and retained hashes, then perform the approved isolated install/UAT/entitlement/dummy-report/Revit workflow checks. Keep the ordinary production desktop process, port 4317, filesystem/state roots, identities, public tags, feeds, and releases untouched. +- The current staging profile intentionally targets production service endpoints, so production-backed UAT does not begin until an independently allocated **dedicated production test tenant/account** is supplied and recorded without secrets. Bind the staging session to that tenant and enforce a checked-in closed request policy enumerating each allowed origin, HTTP method, route/resource prefix, expected read/mutation class, and cleanup action for entitlement, dummy-report, and Revit runs. Reject every request outside the tenant/resource allowlist, snapshot the tenant's authorized baseline, record every mutation/result ID, delete or revert all created test data, and prove post-UAT reconciliation to the baseline. Without that tenant authority and cleanup evidence, UAT stops before the first service request. +- Carry source bundle digests, toolchain/build receipt digest/build ID, and unsigned-test-media status into `distribution.json`, installed marker, structured startup log, and UAT evidence. Carry the pre-pack manifest digest and final package hashes only in the external evidence envelope; boot recomputes/verifies the packaged pre-pack manifest instead of trusting a circular embedded digest. + +## 8. Requested post-UAT integration + +- Only after `.2` UAT passes, invoke `xeorvt-integrate` and merge freshly fetched FloLess `master` and AWARE `main` only into the xeoRVT integration branches. +- After each integration merge, commit the merge result, create and verify new self-contained AWARE and FloLess **post-merge build-input bundles**, and require all `.3` work to consume only those bundles and the closed offline dependency/toolchain inputs. Recompute the canonical pre-pack manifest. If any byte or provenance field changes, mint private `.3`; never overwrite `.2`. Repeat both-builder runtime reproduction when AWARE inputs changed, then combined verification, isolated upgrade, and real UI/UAT. +- Commit only redacted aggregate evidence and hashes after secret scanning. Do not push or merge into product defaults without new explicit authorization. + +## Hard stops + +- No second isolated builder, immutable source bundle, exact toolchain, exclusive build lock, or byte identity: no `.2`. +- No independently controlled signing key: `.2` remains unauthenticated test media and cannot become release/production authority. +- No dedicated production test tenant/request policy/cleanup authority: no production-backed UAT. +- Any mismatch outside the closed transition policy, any evidence mutation, or any production collision: stop and preserve diagnostics. + +## Out of scope + +- Publishing or signing a public AWARE/FloLess release, tag, feed, or installer. +- Changing the existing public AWARE release workflow in this unit. +- Allocating/rotating OAuth, service-role, signing, native-token, NTFS, bootstrap, or control-plane authority. +- Pushing branches or merging xeoRVT work into AWARE `main` or FloLess `master`. diff --git a/docs/superpowers/specs/2026-08-30-reproducible-windows-runtime-review-log.md b/docs/superpowers/specs/2026-08-30-reproducible-windows-runtime-review-log.md new file mode 100644 index 000000000..a5deab2cb --- /dev/null +++ b/docs/superpowers/specs/2026-08-30-reproducible-windows-runtime-review-log.md @@ -0,0 +1,80 @@ +# Plan Review Log: reproducible AWARE Windows runtime + +Started 2026-08-30. MAX_ROUNDS=5. + +## Round 1 — VERDICT: REVISE + +1. **Critical — “two independent builds” are only two worktrees on one mutable host.** They share Git objects, Cargo/npm caches, compiler, linker, Windows SDK, environment, and potential compromise; `/Brepro` cannot close this, especially because `ring` invokes `cc`/MSVC. **Fix:** Build fresh clones on two independently provisioned builders with isolated caches and digest-pinned Rust, Cargo, MSVC/clang, linker, SDK, Node, and npm toolchains. +2. **Critical — the claimed clean-checkout reproducibility cannot be recreated elsewhere because neither source tip is remotely reachable.** AWARE is eight commits ahead of its remote branch and FloLess is forty-two ahead, while the plan forbids pushing. **Fix:** Retain hash-verified Git bundles/source archives for both exact commits, or publish immutable remote refs. +3. **Critical — the plan relies on signatures that the private package does not have.** The approved internal installer permits unsigned output and `vpk pack` receives no signing arguments. A malicious builder can rewrite binaries, receipts, manifests, and the unsigned executable containing the provider hash coherently. **Fix:** Sign an external canonical manifest/attestation with an independently protected key and verify it during packaging and launch, or explicitly treat `.2` as unauthenticated test media. +4. **High — the release-workflow proposal conflicts with its existing tag-version mutation.** It rewrites `Cargo.toml`, leaves `Cargo.lock` unchanged, injects the forbidden Google secret, and builds without `--locked`. **Fix:** Create a separate internal reproducibility entrypoint and require release versions to be committed consistently rather than rewritten during CI. +5. **High — compiler and dependency authority remains ambient and unrecorded.** Build environment overrides and actual MSVC/SDK/esbuild/postject bytes are not closed by the receipt. **Fix:** Use an environment-allowlisting wrapper with absolute digest-checked tools and record every admitted executable/native input. +6. **High — the receipt has no valid route through the FloLess schema or installer.** Runtime schema v1 is exact and installation hard-codes the old layout. **Fix:** Make the receipt build-only or introduce schema v2 and update both verifiers, layout, installer, marker, manifests, and mutations atomically. +7. **High — verification and staging remain vulnerable to TOCTOU and concurrent builders.** Mutable ignored outputs are checked/copied without one exclusive generation lock. **Fix:** Build into a uniquely owned immutable directory, verify/copy through the same handles, and lock through package hashing. +8. **High — the `.1` transition baseline is not anchored.** The retained runtime is ignored/mutable even though an approved manifest hash exists. **Fix:** Verify retained manifest and full package against approved hashes before deriving the transition inventory, then commit the fixed baseline. +9. **High — the post-integration “any packaged byte” rule is unusable while FloLess packaging remains nondeterministic.** FloLess SEA embeds absolute paths and `vpk` is ambient. **Fix:** Base version decisions on a canonical pre-pack payload manifest or make the entire packaging path reproducible. +10. **Medium — dynamic Git-tree materialization omits Windows filename hazards.** **Fix:** Reject case collisions, reserved/normalizing names, LFS pointers, non-UTF-8 names, unexpected attributes, and invalid file types. +11. **Medium — source-wiring tests do not prove the effective build command.** **Fix:** Behaviorally capture the real invocation and test wrong Node/tool digest, poisoned environment, and omitted-lock mutations. +12. **Medium — receipt identity is absent from operational diagnostics.** **Fix:** Carry receipt digest/build ID into `distribution.json`, installed marker, startup logs, UAT evidence, and field-specific errors. + +Reviewer’s simpler safe shape: one dedicated internal build wrapper consuming retained source bundles and a fixed builder image, emitting one externally authenticated attestation and immutable output directory; leave public release and schema v1 untouched until the boundary is implemented end-to-end. + +### Response + +Accepted all twelve findings. The revision now requires two fresh isolated Windows builders rather than worktrees, self-contained verified Git bundles for unreachable commits, a digest-closed environment-allowlisting build wrapper, behavioral command tests, immutable output plus an exclusive end-to-end lock, NTFS-safe Git materialization, and an anchored `.1` baseline. It leaves the public release workflow unchanged. The receipt is deliberately shipped via an atomic schema-v2 change and exposed in operational evidence. FloLess version identity is based on a canonical pre-pack manifest, with private SEA path leakage and ambient Velopack also closed. Because no independent signing authority exists, `.2` is explicitly unauthenticated internal test media and is hard-stopped from production/release use. + +## Round 2 — VERDICT: REVISE + +1. **Critical — provenance hashes are self-referential.** The pre-pack manifest covers `distribution.json`, while `distribution.json` was to contain that manifest digest and the later final package hash. **Fix:** Use an inner payload manifest excluding itself/envelope and record the final package hash only in external evidence. +2. **High — source-bundle order is contradictory.** Pre-edit bundles cannot contain later implementation commits. **Fix:** Distinguish pre-change recovery bundles from final build-input bundles created after each implementation commit. +3. **High — a receipt cannot contain exact absolute commands/environment values and also be path-free/byte-identical.** **Fix:** Canonicalize logical tool IDs and root tokens in the receipt; keep raw per-builder transcripts separate. +4. **High — path independence is not actually tested when two VMs may use identical layouts.** **Fix:** Require deliberately different root identities/lengths and scan artifacts for raw/encoded path forms. +5. **High — the `.1` package anchor has no literal expected container hashes.** **Fix:** State approved NUPKG, Setup, and portable hashes/sizes or drop the claim. +6. **High — schema-v1 compatibility may let `.2` bypass v2 receipt requirements.** **Fix:** `.2` staging/boot accept only v2; isolate v1 in a read-only transition utility. +7. **High — “normalize semantically” is undefined/open-ended.** **Fix:** Permit only validated UTF-8 CRLF→LF with BOM, lone CR, final newline, and every other difference rejected. +8. **Medium — fixed 6,916 test count becomes stale as tests are added.** **Fix:** Run dynamically discovered complete suite, record post-change count, and guard against skipped/lost test files. + +### Response + +Accepted all eight findings. The plan now uses non-circular `prepack-payload-manifest.json` plus an external evidence envelope, splits recovery and final build bundles, tokenizes canonical receipts while retaining raw builder transcripts separately, mandates deliberately different builder paths plus encoded-path scans, pins all three retained `.1` container hashes/sizes, makes `.2` v2-only at stage/install/boot, defines the sole text transition as exact UTF-8 CRLF→LF, and replaces the stale test count with dynamic complete-suite/discovery-loss checks. + +## Round 3 — VERDICT: REVISE + +The eight named Round 2 defects were substantively addressed, but six adjacent execution boundaries remained: + +1. **High — FloLess packaging is not hermetic.** The input list omitted `server/package-lock.json`, while live builds use ambient Node and ignored esbuild/postject/rcedit files. **Fix:** Add a FloLess wrapper that extracts a verified bundle into an empty root, installs offline from the lock, and invokes only digest-bound tools. +2. **High — the Velopack pin authenticates only the launcher, not its 449-file .NET tool closure.** **Fix:** Retain/hash the NUPKG, full resolved tool/runtime closure, and invoke an absolute verified launcher with PATH disabled. +3. **High — retained dependencies are incomplete.** AWARE has 424 registry crates and the reader consumes additional npm/WASM inputs. **Fix:** Vendor/hash every Cargo crate and npm tarball/native artifact, disable network in both builds, and reject lifecycle downloads. +4. **High — post-integration `.3` ordering does not require new bundles.** **Fix:** After every merge, create new verified AWARE/FloLess bundles and build only from them. +5. **High — the exclusive lock has no shared identity/acquisition protocol.** **Fix:** Use a machine-wide package/version-keyed mutex or fixed lock, hold its handle through final hashing, and emit owner/build diagnostics. +6. **Medium — dynamic discovery cannot detect deleted tests by itself.** **Fix:** Compare exact sorted test-file inventory with the recovery bundle plus a reviewed delta, and reject TAP skips outside a committed allowlist. + +### Response + +Accepted all six findings. The plan now adds a separate offline/digest-closed FloLess build wrapper and includes its lock/dependencies; vendors and hashes all Cargo/npm/native/WASM inputs with network disabled; binds the entire Velopack NuGet/.NET/tool closure; requires fresh post-merge bundles; specifies a fail-closed machine-wide version-keyed mutex held by the outer wrapper; and gates the exact test-file inventory plus TAP skip/todo policy against a reviewed recovery-bundle delta. + +## Round 4 — VERDICT: REVISE + +Round 3's six mechanisms were generally feasible, but five blockers remained: + +1. **High — sequential rebuilds can still produce different `.2` media.** A transient mutex does not enforce never-rebuild. **Fix:** Persist a non-deletable completed-version seal containing source, pre-pack, and package hashes; reject every later build of that version. +2. **High — offline dependency closure is generated in the wrong phase.** Pre-change locks cannot authorize final/post-merge dependencies. **Fix:** Generate a new offline closure from each exact final/post-merge bundle and bind it to that builder manifest. +3. **High — retaining .NET does not force Velopack to use it.** A framework-dependent apphost can roll forward/use ambient hooks/stores/runtimes. **Fix:** Use pinned private `dotnet.exe exec` with explicit runtimeconfig/deps, sanitized host environment, disabled roll-forward/multilevel lookup, and loaded-module verification. +4. **High — “without touching production” conflicts with production service endpoints.** **Fix:** Use non-production endpoints or require a dedicated production test tenant with an enumerated request/mutation allowlist and cleanup evidence. +5. **Medium — assertion counts are unavailable from the current runner.** **Fix:** Gate the available file/test/skip/todo/cancel/failure counts or add runtime assertion instrumentation. + +### Response + +Accepted all five findings. The plan now creates a permanent create-new version reservation before mutation; any crash burns the version, and a durable completed seal prevents sequential rebuilds. It regenerates/binds offline closures per final/post-merge bundle, invokes Velopack through a fully pinned private `dotnet exec` module graph, explicitly gates production-backed UAT on a dedicated tenant plus closed request and cleanup ledger, and limits TAP claims to the counts the runner can actually observe. + +## Round 5 — VERDICT: APPROVED + +No concrete implementation blocker remains. Round 4 closes the prior defects: + +- The create-new permanent seal reserves the version before mutation, burns failed versions, prevents concurrent and sequential reuse, and has required deletion/crash mutation tests. +- Offline closures are regenerated from each final or post-merge bundle's exact three lockfiles and bound to that generation. +- Velopack runs through pinned private `dotnet.exe exec` with explicit runtime/dependency files, sanitized host authority, and verified loaded modules. +- Production-backed UAT requires a dedicated tenant, closed request policy, mutation ledger, cleanup, baseline reconciliation, and stops before networking without them. +- The test gate uses exact file inventory and observable TAP counts while explicitly rejecting unsupported assertion-count claims. + +The reviewer concluded that these requirements adequately replace the ambient dependencies, global `vpk`, production endpoint behavior, and unparsed test output present in the live FloLess code.