From 805615905f04ff2ac9ae7c08c507b1e3c30515b4 Mon Sep 17 00:00:00 2001 From: Nate Date: Wed, 14 Oct 2026 02:29:37 -0400 Subject: [PATCH] Reconstruct exact files from declared nested BSA 104 containers --- .github/workflows/windows.yml | 2 +- README.md | 2 + docs/bsa-inspection.md | 2 +- docs/installed-inventory.md | 2 +- docs/loadstone-format.md | 5 +- docs/native-reconstruction.md | 10 + electron/bsa-limits.cjs | 11 + electron/bsa104.cjs | 84 +++++-- electron/native-archive-path.cjs | 17 ++ electron/native-bsa.cjs | 320 +++++++++++++++++++++++++ electron/native-format.cjs | 38 ++- electron/native-materializer.cjs | 52 +++-- tests/bsa.test.mjs | 90 +++++++- tests/native-bsa-electron.test.mjs | 97 ++++++++ tests/native-bsa-fixture.mjs | 73 ++++++ tests/native-bsa.test.mjs | 359 +++++++++++++++++++++++++++++ 16 files changed, 1122 insertions(+), 42 deletions(-) create mode 100644 electron/bsa-limits.cjs create mode 100644 electron/native-archive-path.cjs create mode 100644 electron/native-bsa.cjs create mode 100644 tests/native-bsa-electron.test.mjs create mode 100644 tests/native-bsa-fixture.mjs create mode 100644 tests/native-bsa.test.mjs diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index c05b50b..27b15c5 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -38,4 +38,4 @@ jobs: node scripts/steam-depot.mjs --verify $kit.FullName if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } $env:LOADSTONE_TEST_EXE = Join-Path $kit.FullName 'content/Loadstone.exe' - node --test tests/electron.test.mjs tests/live.test.mjs tests/nexus.test.mjs tests/onboarding.test.mjs tests/close-guard.test.mjs tests/native-preview.test.mjs tests/native-builder.test.mjs tests/native-archives.test.mjs tests/native-library-electron.test.mjs tests/native-guide-electron.test.mjs tests/native-repair-electron.test.mjs tests/native-update-electron.test.mjs tests/native-manual-electron.test.mjs + node --test tests/electron.test.mjs tests/live.test.mjs tests/nexus.test.mjs tests/onboarding.test.mjs tests/close-guard.test.mjs tests/native-preview.test.mjs tests/native-builder.test.mjs tests/native-archives.test.mjs tests/native-bsa-electron.test.mjs tests/native-library-electron.test.mjs tests/native-guide-electron.test.mjs tests/native-repair-electron.test.mjs tests/native-update-electron.test.mjs tests/native-manual-electron.test.mjs diff --git a/README.md b/README.md index ff2c165..62c2530 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,8 @@ The [real TTW acceptance record](docs/native-steam-acceptance.md) now includes d [Manual Nexus downloads](docs/native-manual-downloads.md) open each declared file page in the default browser and verify the downloaded original before selection, with byte progress and cancellation. Browser mode tracks pending files and requires all Nexus selections before installation review; existing cache files can be explicitly checked too. Automatic Nexus downloads still require fresh Premium access. Real account/website acceptance and public API registration remain open. +Recipes can also [reconstruct exact loose files from BSA 104 containers](docs/native-reconstruction.md#nested-bsa-104-files) inside original ZIP/7z/RAR downloads. Each archive, container and inner file has an explicit fingerprint; temporary staging, cancellation, resume and repair preserve the existing installation checks. Automatic nested-source mapping and generated-content reproduction remain unfinished. + ## Persistence and desktop behavior **My library → Repair native setup** now offers [same-version native repair](docs/native-repair.md). It keeps matching files, restores missing/damaged recipe outputs, retains replaced originals and preserves existing profiles/configuration by default. A separate option restores recipe configuration too. Interrupted repairs require fresh review and keep library launch disabled. [Install another version](docs/native-updates.md) creates a separate setup with fresh recipe settings, preserving the original playthrough. Real gameplay acceptance remains open. diff --git a/docs/bsa-inspection.md b/docs/bsa-inspection.md index f46a2a6..24cde0b 100644 --- a/docs/bsa-inspection.md +++ b/docs/bsa-inspection.md @@ -14,6 +14,6 @@ This initial reader accepts the version-104 layout used by Fallout 3/New Vegas: Limits: 4,294,967,295 input bytes, 64 MiB metadata, 100,000 folders, 500,000 files, 1 GB per uncompressed member, 20 GB total logical member content (including shared copies) and five minutes per inspection. Hashing streams have bounded buffers; declared sizes, zlib input consumption and output counts are verified. The reader uses the same open file handle throughout and checks file identity afterward. -This report is **source evidence only**. It does not change the ZIP/7z/RAR ledger or add a native recipe operation. To establish an upstream source chain, separately verify the outer download archive, its exact BSA member hash and the desired internal member hash. Compare the installed file's complete size/SHA-256; a similar name or image is insufficient. Native nested-BSA reconstruction, transformed-file reproduction, rights review and real installation acceptance remain separate work. No game or mod data is bundled by this command. +This report is **source evidence only**. It does not change the ZIP/7z/RAR ledger or automatically produce a recipe. To establish an upstream source chain, separately verify the outer download archive, its exact BSA member hash and the desired internal member hash. Compare the installed file's complete size/SHA-256; a similar name or image is insufficient. A reviewed recipe can use the explicit [`extract-bsa104` action](loadstone-format.md#outputs-and-actions) for [bounded nested-BSA reconstruction](native-reconstruction.md#nested-bsa-104-files). Transformed-file reproduction, rights review and real installation acceptance remain separate work. No game or mod data is bundled by this command. The JavaScript implementation and synthetic fixtures are original. Format semantics were checked against Wabbajack's pinned [104/105 reader](https://github.com/wabbajack-tools/wabbajack/blob/4.2.3.0/Wabbajack.Compression.BSA/TES5Archive/Reader.cs), [folder records](https://github.com/wabbajack-tools/wabbajack/blob/4.2.3.0/Wabbajack.Compression.BSA/TES5Archive/FolderRecord.cs) and [file records](https://github.com/wabbajack-tools/wabbajack/blob/4.2.3.0/Wabbajack.Compression.BSA/TES5Archive/FileRecord.cs). Loadstone deliberately supports a narrower format subset. diff --git a/docs/installed-inventory.md b/docs/installed-inventory.md index 7cbedae..f07a1cf 100644 --- a/docs/installed-inventory.md +++ b/docs/installed-inventory.md @@ -84,7 +84,7 @@ For tests, `scripts/prepare-sevenzip.ps1 -Destination ` d 7-Zip is a separate upstream tool with LGPL, BSD and unRAR terms; a future bundled distribution needs its own notices/source compliance review. See the [official release](https://github.com/ip7z/7zip/releases/tag/26.04) and [license](https://www.7-zip.org/license.txt). -For a separate read-only inventory of files inside a local Fallout 3/New Vegas BSA, see [BSA 104 inspection](bsa-inspection.md). This hashes the internal members but does not yet add nested-container matches to the ledger or native reconstruction. +For a separate read-only inventory of files inside a local Fallout 3/New Vegas BSA, see [BSA 104 inspection](bsa-inspection.md). This hashes the internal members without adding nested-container matches to the ledger. Reviewed source chains can be declared explicitly for [native BSA reconstruction](native-reconstruction.md#nested-bsa-104-files). ## Native `.loadstone` format diff --git a/docs/loadstone-format.md b/docs/loadstone-format.md index 3b9ef44..645b9ed 100644 --- a/docs/loadstone-format.md +++ b/docs/loadstone-format.md @@ -80,7 +80,8 @@ Copy/extract outputs are `{path, bytes, sha256, action}`, optionally with `mtime Full recipe verification and build recovery require both the declared bytes/hash and timestamp. A timestamp-only change becomes a recovery/repair conflict: preserve the old file before replacing it, and retain it during removal. Repair's existing default of preserving personal configuration still applies; explicitly restoring configuration is needed to recover its recipe metadata. Normal launch retains its existing executable/profile safety checks and does not perform full recipe verification. Omission preserves legacy behavior and canonical recipe identity. Adding the field creates a new recipe fingerprint; earlier readers reject the unsupported field rather than silently ignoring it. - `{type: "copy", source: "source-id"}` copies a `game-file`, `prerequisite-file` or `payload`. Expected output bytes/hash must equal the input. -- `{type: "extract", source: "archive-id", member: "Data/Example.esp"}` selects one safe member of a locked Nexus/HTTPS archive. The output hash is the hash of the extracted member. Nested archives, FOMOD execution, binary patches and arbitrary text substitution are not implemented by this version. +- `{type: "extract", source: "archive-id", member: "Data/Example.esp"}` selects one safe member of a locked Nexus/HTTPS archive. The output hash is the hash of the extracted member. It never recursively interprets extracted bytes. +- `{type: "extract-bsa104", source: "archive-id", container: {member: "Data/Example.bsa", bytes: 12345, sha256: "<64 lowercase hex characters>"}, member: "meshes/example.nif"}` explicitly selects one internal member of a BSA-104 container within a locked Nexus/HTTPS ZIP, 7z or RAR download. The container has its own exact size/hash; the output locks the inner file's exact uncompressed size/hash. Both member paths follow the canonical recipe path rules. A recipe may use at most 64 distinct containers totaling 20 GB of staging data; each container is 36–4,294,967,295 bytes and each inner output is at most 1 GB. Conflicting declarations for the same container are rejected. Only the [documented BSA-104 subset](bsa-inspection.md) is accepted. This is exactly one declared container level; direct owned-game BSA inputs, BSA writing, BA2, BSA 103/105, arbitrary nested archives, FOMOD execution, binary patches and arbitrary text substitution remain unsupported. - `{type: "mo2-config", gameEdition: "Steam", executables: [...]}` generates only `ModOrganizer.ini` from bounded, relative executable declarations and recipe game/launch fields. `gameEdition` accepts `Steam` or `GOG` matching the target game requirement; executable entries may declare `steamAppId` matching the supported target game and its Steam requirement. Both fields are optional for legacy recipes, whose exact generated bytes remain unchanged. It has no source or static output fingerprint. The materializer computes the exact destination-specific bytes/hash before writing and records them in the receipt after verification. An empty, fingerprinted `portable.txt` output is required. See the [complete generator contract](native-mo2-configuration.md), including serialization, size bounds and unsupported settings. Outputs must include `ModOrganizer.exe`, `ModOrganizer.ini`, the declared profile's `modlist.txt` and `plugins.txt`, and the target game executable beneath `launch.gameRoot`. Mod priorities, disabled entries and plugin order are carried by the exact profile files. Where the manager/game derives order from modification times, declare the corresponding winning plugin timestamps too. Neither order nor timestamps are inferred from source order. Final-file verification checks each static or derived output fingerprint after reconstruction. Save paths may never be claimed as owned recipe outputs. @@ -98,6 +99,6 @@ The compiler validates and hashes all bundled bytes, creates the output exclusiv ## Next implementation gates -An [experimental local materializer](native-reconstruction.md) reconstructs copy and ZIP/7z/RAR-extract recipes in a new isolated directory and verifies every output hash. It is separate from the desktop queue and does not establish playability. +The [experimental native materializer](native-reconstruction.md) reconstructs copy, ZIP/7z/RAR-extract and explicitly declared BSA-104 recipes in an isolated directory and verifies every output hash. The desktop guide and creator tools use this materializer; reconstruction does not establish playability. Complete installed-file/source mapping; add bounded reconstruction for edited/generated files; verify real provider downloads and extend supported source flows; extend durable creator jobs with partial-build recovery and playable-library integration. Preserve the existing launch/shortcut/removal ownership checks. Prove actual clean installation and gameplay before publishing the three featured recipes or assigning 1.0. diff --git a/docs/native-reconstruction.md b/docs/native-reconstruction.md index 0571ac1..ac7bd97 100644 --- a/docs/native-reconstruction.md +++ b/docs/native-reconstruction.md @@ -55,6 +55,16 @@ Cancellation or failure retains the candidate directory and receipt. A hard proc These checks supplement standard Windows access controls; they are not a sandbox for an actively hostile process racing filesystem changes. Keep source files and the new destination settled during reconstruction. +## Nested BSA 104 files + +An explicit `extract-bsa104` action can reconstruct loose files from a BSA container inside an original ZIP, 7z or RAR source. The recipe independently locks the outer archive, its BSA member and the final inner file. The installer validates the complete outer archive structure, stages only the declared containers, verifies their full hashes, and checks every BSA member and selected output before creating or changing installation files. Ordinary `extract` actions keep their direct-member meaning. + +Staging uses one new directory beneath the operating system's temporary directory and numbered files, never archive-supplied filesystem paths. Up to 64 containers and 20 GB of staging bytes are accepted per recipe. The temporary volume needs the declared container bytes plus a 64 MB reserve; destination space is checked separately. Each container follows the [BSA reader's format and decompression limits](bsa-inspection.md). A compressed or shared member is streamed through an awaited consumer, then the installation writer verifies its exact output bytes/hash and applies any declared timestamp. No archive code runs. + +Handled success, cancellation and failure close the readers/writers and remove the operation's ordinary staging files. Cleanup checks directory/file identity, performs no recursive deletion and preserves changed or unfamiliar staging contents. A hard process termination can leave temporary containers behind; restarting builds new staging after a fresh source/recovery review and does not blindly purge earlier temporary folders. Failed installation outputs and user data retain the existing receipt/recovery behavior. Resume reuses verified outputs; repair preserves conflicts and personal configuration through the same review path as other actions. + +Synthetic tests cover ZIP, solid 7z and RAR5 outer archives, mixed direct/nested outputs, compressed/empty/shared BSA members, exact timestamps, invalid containers and inner members, unsafe trailing archive entries, resource limits, failed staging writes, cancellation, resume and repair. Actual Electron tests exercise reviewed ZIP and RAR5 reconstruction through the bridge and are also run against the staged executable. The CLI inventory remains evidence only; automatic nested-source ledger mapping is separate work. + ## Acceptance remaining Complete real provider-download acceptance, conversion, profile/tool configuration reconstruction, real version-update and maintenance acceptance. Creator reconstruction is exercised through real Electron with original synthetic ZIP, solid 7z and stored RAR5 inputs, including cancellation, failed writes and tool replacement. The RAR5 fixture is generated from the [published format](https://www.rarlab.com/technote.htm), without third-party archive contents or encoder code. Clean-machine installation and gameplay acceptance remain unfinished. The separate package inspection card stays read-only. diff --git a/electron/bsa-limits.cjs b/electron/bsa-limits.cjs new file mode 100644 index 0000000..fc2ed27 --- /dev/null +++ b/electron/bsa-limits.cjs @@ -0,0 +1,11 @@ +module.exports = Object.freeze({ + archiveBytes: 0xffffffff, + metadataBytes: 64 * 1024 * 1024, + folders: 100000, + files: 500000, + memberBytes: 1_000_000_000, + totalBytes: 20_000_000_000, + milliseconds: 300000, + containers: 64, + stagingBytes: 20_000_000_000, +}); diff --git a/electron/bsa104.cjs b/electron/bsa104.cjs index 0a6bede..656a907 100644 --- a/electron/bsa104.cjs +++ b/electron/bsa104.cjs @@ -6,15 +6,7 @@ const { createInflate } = require('node:zlib'); const { ordinaryFile, sameFile } = require('./native-package.cjs'); const { relativeFile } = require('./manifest.cjs'); -const limits = Object.freeze({ - archiveBytes: 0xffffffff, - metadataBytes: 64 * 1024 * 1024, - folders: 100000, - files: 500000, - memberBytes: 1_000_000_000, - totalBytes: 20_000_000_000, - milliseconds: 300000, -}); +const limits = require('./bsa-limits.cjs'); function demand(ok, message) { if (!ok) throw new Error(`BSA inspection: ${message}`); } @@ -33,8 +25,9 @@ function normalizedName(bytes, folder = false) { } // Original, deliberately limited BSA 104 reader. See docs/bsa-inspection.md -// for format references and unsupported variants. No payload is written to disk. -async function inspectBsa104(file, options = {}) { +// for format references and unsupported variants. Filesystem destinations belong +// to the trusted consumer, never to paths supplied by the BSA. +async function readBsa104(file, options = {}, extraction) { const signal = options.signal ? AbortSignal.any([ options.signal, @@ -79,7 +72,13 @@ async function inspectBsa104(file, options = {}) { ); return bytes; } - async function digestRange(offset, storedBytes, bytes, compressed = false) { + async function digestRange( + offset, + storedBytes, + bytes, + compressed = false, + consume, + ) { signal.throwIfAborted(); const digest = crypto.createHash('sha256'); let count = 0; @@ -98,7 +97,11 @@ async function inspectBsa104(file, options = {}) { new Error('BSA inspection: output exceeds declared size.'), ); digest.update(chunk); - callback(); + if (consume) + Promise.resolve() + .then(() => consume(chunk)) + .then(() => callback(), callback); + else callback(); }, }); if (compressed) { @@ -268,19 +271,41 @@ async function inspectBsa104(file, options = {}) { } const hashed = new Map(), members = []; + if (extraction) { + const indexed = new Map(entries.map((entry) => [entry.path, entry])); + for (const [name, expectedMember] of extraction.selected) + demand( + indexed.get(name)?.bytes === expectedMember.bytes, + 'selected member is missing or differs in size.', + ); + } for (const entry of entries) { signal.throwIfAborted(); + const selected = extraction?.selected.get(entry.path); + if (extraction && !selected) continue; + const publicEntry = { name: entry.path, bytes: entry.bytes }; const key = `${entry.offset}:${entry.storedBytes}:${entry.compressed}`; - let memberHash = hashed.get(key); + let memberHash = extraction ? undefined : hashed.get(key); if (!memberHash) { memberHash = await digestRange( entry.start, entry.length, entry.bytes, entry.compressed, + extraction + ? (chunk) => extraction.consumer.consume(publicEntry, chunk) + : undefined, ); hashed.set(key, memberHash); } + if (extraction) { + demand( + memberHash === selected.sha256, + 'selected member differs from expected hash.', + ); + signal.throwIfAborted(); + await extraction.consumer.finish(publicEntry); + } members.push({ path: entry.path, bytes: entry.bytes, @@ -310,4 +335,33 @@ async function inspectBsa104(file, options = {}) { fs.closeSync(fd); } } -module.exports = { inspectBsa104, limits }; +async function inspectBsa104(file, options = {}) { + return readBsa104(file, options); +} +async function extractBsa104(file, expected, wanted, consumer, signal) { + demand( + expected && + wanted instanceof Map && + wanted.size > 0 && + wanted.size <= limits.files && + typeof consumer?.consume === 'function' && + typeof consumer?.finish === 'function', + 'invalid extraction request.', + ); + const selected = new Map(); + for (const [name, value] of wanted) { + demand( + relativeFile(name) && + !name.includes('\\') && + Number.isSafeInteger(value?.bytes) && + value.bytes >= 0 && + value.bytes <= limits.memberBytes && + typeof value.sha256 === 'string' && + /^[a-f0-9]{64}$/.test(value.sha256), + 'invalid selected member fingerprint.', + ); + selected.set(name, { bytes: value.bytes, sha256: value.sha256 }); + } + await readBsa104(file, { expected, signal }, { selected, consumer }); +} +module.exports = { inspectBsa104, extractBsa104, limits }; diff --git a/electron/native-archive-path.cjs b/electron/native-archive-path.cjs new file mode 100644 index 0000000..fe9039b --- /dev/null +++ b/electron/native-archive-path.cjs @@ -0,0 +1,17 @@ +const { relativeFile } = require('./manifest.cjs'); +function zipPath(entry) { + const directory = entry.fileName.endsWith('/'), + name = directory ? entry.fileName.slice(0, -1) : entry.fileName; + const type = (entry.externalFileAttributes >>> 16) & 0o170000; + if ( + !relativeFile(name) || + name.includes('\\') || + name !== name.normalize('NFC') || + (type && type !== (directory ? 0o040000 : 0o100000)) || + entry.generalPurposeBitFlag & 1 || + ![0, 8].includes(entry.compressionMethod) + ) + throw new Error('Unsafe or unsupported ZIP entry.'); + return { name, directory }; +} +module.exports = { zipPath }; diff --git a/electron/native-bsa.cjs b/electron/native-bsa.cjs new file mode 100644 index 0000000..0cc0f6d --- /dev/null +++ b/electron/native-bsa.cjs @@ -0,0 +1,320 @@ +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const yauzl = require('yauzl'); +const { ordinaryFile, sameFile } = require('./native-package.cjs'); +const { canonical, contains } = require('./live-catalog.cjs'); +const { inspectBsa104, limits } = require('./bsa104.cjs'); +const { zipPath } = require('./native-archive-path.cjs'); +const { nativeActivity } = require('./native-activity.cjs'); + +function guardArchive(group) { + if ( + !sameFile( + group.input.snapshot, + ordinaryFile(group.input.file, 50_000_000_000), + ) + ) + throw new Error('BSA source archive changed after input verification.'); +} +async function inspectOuter(group, options) { + guardArchive(group); + if (group.input.inspectedArchive) { + const { selected } = options.sevenZip.planExtraction( + group.input.inspectedArchive, + new Set(group.members.keys()), + ); + if ( + selected.some( + (entry) => group.members.get(entry.name).expected.bytes !== entry.bytes, + ) + ) + throw new Error('BSA container size differs from its archive member.'); + return; + } + const zip = await yauzl.openPromise(group.input.file, { + lazyEntries: true, + autoClose: false, + strictFileNames: false, + validateEntrySizes: true, + }); + const paths = new Map(), + found = new Set(); + let total = 0; + try { + for await (const entry of zip.eachEntry()) { + options.signal?.throwIfAborted(); + const { name, directory } = zipPath(entry), + key = name.toLowerCase(); + if (paths.has(key)) throw new Error('Duplicate BSA source archive path.'); + paths.set(key, directory ? 'directory' : 'file'); + total += entry.uncompressedSize; + if (paths.size > 500000 || total > 100_000_000_000) + throw new Error('BSA source archive exceeds limits.'); + const wanted = group.members.get(name); + if (wanted) { + if (directory || wanted.expected.bytes !== entry.uncompressedSize) + throw new Error( + 'BSA container size differs from its archive member.', + ); + found.add(name); + } + } + for (const name of paths.keys()) { + const parts = name.split('/'); + while (parts.length > 1) { + parts.pop(); + if (paths.get(parts.join('/')) === 'file') + throw new Error('BSA source archive paths overlap.'); + } + } + if (found.size !== group.members.size) + throw new Error('Required BSA container is missing.'); + } finally { + zip.close(); + } + guardArchive(group); +} + +// One operation owns one new temporary directory and numbered files. It never +// uses an archive name as a filesystem destination or recursively removes data. +class NativeBsaWork { + constructor() { + this.root = null; + this.items = []; + } + guard() { + const stat = fs.lstatSync(this.root); + if ( + !stat.isDirectory() || + stat.isSymbolicLink() || + stat.ino !== this.identity.ino || + stat.dev !== this.identity.dev || + fs.realpathSync(this.root).toLowerCase() !== this.root.toLowerCase() + ) + throw new Error('BSA staging directory changed.'); + } + async prepare(outputs, sources, destination, options) { + const groups = new Map(); + for (const output of outputs) { + const action = output.action; + if (action.type !== 'extract-bsa104') continue; + const input = sources.get(action.source); + if (!input?.archive) + throw new Error('BSA extraction requires an original archive.'); + const group = groups.get(input.file) || { input, members: new Map() }; + const { container } = action; + let item = group.members.get(container.member); + if ( + item && + (item.expected.bytes !== container.bytes || + item.expected.sha256 !== container.sha256) + ) + throw new Error('Conflicting BSA container fingerprints.'); + if (!item) { + item = { + expected: { bytes: container.bytes, sha256: container.sha256 }, + outputs: new Map(), + }; + group.members.set(container.member, item); + this.items.push(item); + } + const matches = item.outputs.get(action.member) || []; + if ( + matches.some( + (v) => v.bytes !== output.bytes || v.sha256 !== output.sha256, + ) + ) + throw new Error('Conflicting BSA member fingerprints.'); + matches.push(output); + item.outputs.set(action.member, matches); + groups.set(input.file, group); + } + if (!this.items.length) return []; + const bytes = this.items.reduce( + (sum, item) => sum + item.expected.bytes, + 0, + ); + if (this.items.length > limits.containers || bytes > limits.stagingBytes) + throw new Error('BSA staging exceeds limits.'); + for (const group of groups.values()) await inspectOuter(group, options); + options.signal?.throwIfAborted(); + const parent = fs.realpathSync(os.tmpdir()), + space = fs.statfsSync(parent, { bigint: true }); + if (space.bavail * space.bsize < BigInt(bytes) + 64_000_000n) + throw new Error('Insufficient temporary disk space for BSA containers.'); + this.root = fs.mkdtempSync(path.join(parent, 'loadstone-bsa-')); + this.identity = fs.lstatSync(this.root); + if ( + contains(canonical(destination), this.root) || + contains(this.root, canonical(destination)) + ) + throw new Error('BSA staging overlaps the installation destination.'); + this.items.forEach((item, index) => { + item.file = path.join(this.root, `${index}.bsa`); + }); + const activity = nativeActivity( + options.activity, + 'archives', + this.items.length * 2, + ); + for (const group of groups.values()) { + this.guard(); + guardArchive(group); + let current; + const begin = async (entry) => { + options.signal?.throwIfAborted(); + this.guard(); + const item = group.members.get(entry.name); + if (!item || item.expected.bytes !== entry.bytes) + throw new Error('Unexpected BSA staging member.'); + if (!current) { + const handle = await fs.promises.open(item.file, 'wx'); + current = { + item, + handle, + bytes: 0, + digest: crypto.createHash('sha256'), + }; + item.identity = await handle.stat(); + activity.start(item.expected.bytes); + } + if (current.item !== item) + throw new Error('BSA staging member order changed.'); + return current; + }; + const consumer = { + consume: async (entry, chunk) => { + const item = await begin(entry); + item.bytes += chunk.length; + if (item.bytes > item.item.expected.bytes) + throw new Error('BSA container exceeds declared size.'); + item.digest.update(chunk); + activity.add(chunk.length); + let offset = 0; + while (offset < chunk.length) { + options.signal?.throwIfAborted(); + const { bytesWritten } = await item.handle.write( + chunk, + offset, + chunk.length - offset, + ); + if (!bytesWritten) + throw new Error('Could not write BSA staging file.'); + offset += bytesWritten; + } + }, + finish: async (entry) => { + const item = await begin(entry); + await item.handle.sync(); + await item.handle.close(); + item.handle = null; + if ( + item.bytes !== item.item.expected.bytes || + item.digest.digest('hex') !== item.item.expected.sha256 + ) + throw new Error( + 'BSA container differs from its declared fingerprint.', + ); + item.item.snapshot = ordinaryFile( + item.item.file, + item.item.expected.bytes, + ); + current = null; + activity.finish(); + }, + }; + try { + if (group.input.inspectedArchive) + await options.sevenZip.extractMembers( + group.input.candidate, + group.input.inspectedArchive, + new Set(group.members.keys()), + consumer, + options.signal, + ); + else { + const zip = await yauzl.openPromise(group.input.file, { + lazyEntries: true, + autoClose: false, + strictFileNames: false, + validateEntrySizes: true, + }); + const produced = new Set(); + try { + for await (const entry of zip.eachEntry()) { + options.signal?.throwIfAborted(); + const { name, directory } = zipPath(entry); + if (directory || !group.members.has(name)) continue; + if (produced.has(name)) + throw new Error('Duplicate BSA staging member.'); + produced.add(name); + const selected = { name, bytes: entry.uncompressedSize }; + const stream = await zip.openReadStreamPromise(entry); + for await (const chunk of stream) + await consumer.consume(selected, chunk); + await consumer.finish(selected); + } + if (produced.size !== group.members.size) + throw new Error('BSA staging member was not produced.'); + } finally { + zip.close(); + } + } + } finally { + await current?.handle?.close(); + } + guardArchive(group); + } + for (const item of this.items) { + this.guard(); + activity.start(0); + const inspected = await inspectBsa104(item.file, { + expected: item.expected, + signal: options.signal, + }); + const members = new Map( + inspected.members.map((member) => [member.path, member]), + ); + for (const [name, outputs] of item.outputs) { + const member = members.get(name); + if ( + !member || + outputs.some( + (output) => + member.bytes !== output.bytes || member.sha256 !== output.sha256, + ) + ) + throw new Error('BSA member differs from the declared output.'); + } + activity.finish(); + } + return this.items.map((item) => ({ + file: item.file, + snapshot: item.snapshot, + bsa: item.expected, + outputs: item.outputs, + })); + } + close() { + if (!this.root) return; + this.guard(); + for (const item of this.items) { + if (!item.file || !fs.existsSync(item.file)) continue; + this.guard(); + const stat = ordinaryFile(item.file, limits.archiveBytes); + if ( + !item.identity || + stat.ino !== item.identity.ino || + stat.dev !== item.identity.dev + ) + throw new Error('BSA staging file changed before cleanup.'); + fs.unlinkSync(item.file); + } + this.guard(); + fs.rmdirSync(this.root); + this.root = null; + } +} +module.exports = { NativeBsaWork }; diff --git a/electron/native-format.cjs b/electron/native-format.cjs index cc38688..f97fb24 100644 --- a/electron/native-format.cjs +++ b/electron/native-format.cjs @@ -1,5 +1,6 @@ const { relativeFile } = require('./manifest.cjs'); const { configBytes, validateMo2Config } = require('./native-mo2-config.cjs'); +const bsaLimits = require('./bsa-limits.cjs'); const limits = Object.freeze({ recipeBytes: 64_000_000, @@ -302,7 +303,8 @@ function validateRecipe(value) { ); array(value.outputs, limits.outputs, 1); const outputs = uniquePaths(value.outputs), - used = new Set(); + used = new Set(), + bsaContainers = new Map(); let installBytes = 0; for (const output of value.outputs) { if (output.action?.type === 'mo2-config') { @@ -347,6 +349,33 @@ function validateRecipe(value) { filePath(action.member), 'extract requires a locked archive and safe member path.', ); + } else if (action.type === 'extract-bsa104') { + shape(action, ['type', 'source', 'container', 'member']); + shape(action.container, ['member', 'bytes', 'sha256']); + fingerprint(action.container); + demand( + ['nexus-archive', 'https-archive'].includes(source.type) && + filePath(action.container.member) && + filePath(action.member) && + action.container.bytes >= 36 && + action.container.bytes <= bsaLimits.archiveBytes && + output.bytes <= bsaLimits.memberBytes, + 'BSA extraction requires a locked archive, container fingerprint and safe member paths.', + ); + const key = `${source.id}:${action.container.member.toLowerCase()}`; + const prior = bsaContainers.get(key); + demand( + !prior || + (prior.member === action.container.member && + prior.bytes === action.container.bytes && + prior.sha256 === action.container.sha256), + 'conflicting BSA container declarations.', + ); + bsaContainers.set(key, action.container); + demand( + bsaContainers.size <= bsaLimits.containers, + 'too many BSA containers.', + ); } else demand( false, @@ -354,6 +383,13 @@ function validateRecipe(value) { ); installBytes += output.bytes; } + demand( + [...bsaContainers.values()].reduce( + (sum, container) => sum + container.bytes, + 0, + ) <= bsaLimits.stagingBytes, + 'BSA staging exceeds supported bounds.', + ); demand( Number.isSafeInteger(installBytes) && bytes(installBytes), 'output storage exceeds supported bounds.', diff --git a/electron/native-materializer.cjs b/electron/native-materializer.cjs index 91b5d71..35427c9 100644 --- a/electron/native-materializer.cjs +++ b/electron/native-materializer.cjs @@ -12,7 +12,9 @@ const { } = require('./native-package.cjs'); const { canonicalJson } = require('./native-format.cjs'); const { canonical, contains } = require('./live-catalog.cjs'); -const { relativeFile } = require('./manifest.cjs'); +const { zipPath } = require('./native-archive-path.cjs'); +const { NativeBsaWork } = require('./native-bsa.cjs'); +const { extractBsa104 } = require('./bsa104.cjs'); const { replaceStateFile } = require('./replace-state-file.cjs'); const { archiveKind } = require('./archive-kind.cjs'); const { @@ -54,21 +56,6 @@ async function verifyFile(file, expected, signal, activity) { activity?.finish(); return before; } -function zipPath(entry) { - const directory = entry.fileName.endsWith('/'), - name = directory ? entry.fileName.slice(0, -1) : entry.fileName; - const type = (entry.externalFileAttributes >>> 16) & 0o170000; - if ( - !relativeFile(name) || - name.includes('\\') || - name !== name.normalize('NFC') || - (type && type !== (directory ? 0o040000 : 0o100000)) || - entry.generalPurposeBitFlag & 1 || - ![0, 8].includes(entry.compressionMethod) - ) - throw new Error('Unsafe or unsupported ZIP entry.'); - return { name, directory }; -} async function inspectInputs(file, options) { const packageSnapshot = ordinaryFile(file, 512_000_000); const inspected = await inspectNativePackage(file), @@ -187,7 +174,7 @@ async function inspectInputs(file, options) { } return { inspected, gameRoots, prerequisiteRoots, sources }; } -async function materializeNative(file, destination, options = {}) { +async function materializeNativeImpl(file, destination, options, bsaWork) { options.signal?.throwIfAborted(); file = path.resolve(file); if (typeof destination !== 'string' || !path.isAbsolute(destination)) @@ -260,7 +247,8 @@ async function materializeNative(file, destination, options = {}) { } // Reject unsupported archive structure before creating the destination. Hash // verification already locked each selected archive's exact bytes. - const groups = new Map(); + const bsaGroups = await bsaWork.prepare(outputs, sources, target, options); + const groups = new Map(bsaGroups.map((group) => [group.file, group])); for (const output of outputs) { if (generated.has(output.path)) continue; const input = sources.get(output.action.source); @@ -286,6 +274,10 @@ async function materializeNative(file, destination, options = {}) { for (const group of groups.values()) { if (!sameFile(group.snapshot, ordinaryFile(group.file, 50_000_000_000))) throw new Error('Archive changed after input verification.'); + if (group.bsa) { + archiveActivity.finish(); + continue; + } if (group.inspectedArchive) { const { selected } = options.sevenZip.planExtraction( group.inspectedArchive, @@ -491,6 +483,18 @@ async function materializeNative(file, destination, options = {}) { // Only Loadstone opens output paths. One file is open at a time, and each // write is awaited before reading more bytes from the archive process. let current; + const extract = group.bsa + ? (_candidate, _inspected, wanted, consumer, signal) => + extractBsa104( + group.file, + group.bsa, + new Map( + [...wanted].map((name) => [name, group.outputs.get(name)[0]]), + ), + consumer, + signal, + ) + : (...args) => options.sevenZip.extractMembers(...args); async function begin(entry) { options.signal?.throwIfAborted(); if (!current) { @@ -511,7 +515,7 @@ async function materializeNative(file, destination, options = {}) { return current; } try { - await options.sevenZip.extractMembers( + await extract( group.candidate, group.inspectedArchive, new Set(group.outputs.keys()), @@ -612,7 +616,7 @@ async function materializeNative(file, destination, options = {}) { for (const group of groups.values()) { if (!sameFile(group.snapshot, ordinaryFile(group.file, 50_000_000_000))) throw new Error('Source archive changed before extraction.'); - if (group.inspectedArchive) { + if (group.inspectedArchive || group.bsa) { await extractWithTool(group); continue; } @@ -685,4 +689,12 @@ async function materializeNative(file, destination, options = {}) { throw error; } } +async function materializeNative(file, destination, options = {}) { + const bsaWork = new NativeBsaWork(); + try { + return await materializeNativeImpl(file, destination, options, bsaWork); + } finally { + bsaWork.close(); + } +} module.exports = { materializeNative }; diff --git a/tests/bsa.test.mjs b/tests/bsa.test.mjs index a659e7b..a25a22f 100644 --- a/tests/bsa.test.mjs +++ b/tests/bsa.test.mjs @@ -8,7 +8,7 @@ import { spawnSync } from 'node:child_process'; import { createRequire } from 'node:module'; import { bsaFixture } from './bsa-fixture.mjs'; const require = createRequire(import.meta.url); -const { inspectBsa104 } = require('../electron/bsa104.cjs'); +const { inspectBsa104, extractBsa104 } = require('../electron/bsa104.cjs'); const fingerprint = (bytes) => ({ bytes: Buffer.byteLength(bytes), sha256: crypto.createHash('sha256').update(bytes).digest('hex'), @@ -253,3 +253,91 @@ test('BSA CLI writes only a new private report after success and protects source assert.notEqual(run(failed).status, 0); assert.equal(fs.existsSync(failed), false); }); + +test('BSA extraction awaits consumers, validates selected hashes and stops writes on cancellation or failure', async (t) => { + const data = Buffer.alloc(500000, 81); + const encoded = bsaFixture([{ path: 'data.bin', data, compressed: true }]); + const { input } = fixture(t, encoded), + expected = fingerprint(encoded.bytes); + const wanted = new Map([['data.bin', fingerprint(data)]]); + const chunks = []; + let active = 0, + finished = 0; + await extractBsa104(input, expected, wanted, { + async consume(entry, chunk) { + assert.equal(active++, 0); + assert.equal(entry.name, 'data.bin'); + await new Promise((resolve) => setTimeout(resolve, 1)); + chunks.push(Buffer.from(chunk)); + active--; + }, + async finish() { + assert.equal(active, 0); + finished++; + }, + }); + assert.deepEqual(Buffer.concat(chunks), data); + assert.equal(finished, 1); + for (const mode of ['cancel', 'failure']) { + const controller = new AbortController(); + let writes = 0, + finishes = 0; + await assert.rejects( + extractBsa104( + input, + expected, + wanted, + { + async consume() { + writes++; + await new Promise((resolve) => setTimeout(resolve, 1)); + if (mode === 'cancel') controller.abort(); + else throw Error('consumer failed'); + }, + async finish() { + finishes++; + }, + }, + controller.signal, + ), + ); + const stopped = writes; + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(writes, stopped); + assert.equal(finishes, 0); + } + let calls = 0; + await assert.rejects( + extractBsa104( + input, + expected, + new Map([['missing.bin', fingerprint(data)]]), + { + consume() { + calls++; + }, + finish() { + calls++; + }, + }, + ), + /missing/, + ); + assert.equal(calls, 0); + await assert.rejects( + extractBsa104( + input, + expected, + new Map([['data.bin', { bytes: data.length, sha256: '0'.repeat(64) }]]), + { + consume() {}, + finish() { + calls++; + }, + }, + ), + /expected hash/, + ); + assert.equal(calls, 0); + assert.deepEqual(fs.readFileSync(input), encoded.bytes); +}); diff --git a/tests/native-bsa-electron.test.mjs b/tests/native-bsa-electron.test.mjs new file mode 100644 index 0000000..183142f --- /dev/null +++ b/tests/native-bsa-electron.test.mjs @@ -0,0 +1,97 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { _electron as electron } from 'playwright'; +import { createNativeBsaFixture } from './native-bsa-fixture.mjs'; +import { fingerprint } from './native-archive-fixture.mjs'; +const require = createRequire(import.meta.url); + +for (const kind of ['zip-windows', 'rar5']) + test( + `real Electron reviews and reconstructs nested BSA104 files from ${kind}`, + { + timeout: 60000, + skip: kind !== 'zip-windows' && !process.env.LOADSTONE_TEST_7ZIP, + }, + async () => { + const f = await createNativeBsaFixture(kind); + const env = { ...process.env, LOADSTONE_DATA_DIR: f.state }; + delete env.ELECTRON_RUN_AS_NODE; + const app = await electron.launch({ + executablePath: process.env.LOADSTONE_TEST_EXE || require('electron'), + args: process.env.LOADSTONE_TEST_EXE ? [] : [path.resolve('.')], + env, + }); + try { + const page = await app.firstWindow(); + await page + .getByRole('button', { name: 'Explore the demo', exact: true }) + .click(); + await page + .getByRole('button', { name: 'Settings', exact: true }) + .click(); + await page.evaluate(() => + window.loadstone.settings({ dummyData: false }), + ); + await app.evaluate( + ({ dialog }, f) => { + process.getBuiltinModule('os').tmpdir = () => f.staging; + const files = [ + f.packageFile, + f.inputsFile, + f.parent, + ...(f.executable ? [f.executable] : []), + ]; + dialog.showOpenDialog = async () => ({ + canceled: false, + filePaths: [files.shift()], + }); + }, + { + ...f, + executable: + kind === 'zip-windows' + ? undefined + : process.env.LOADSTONE_TEST_7ZIP, + }, + ); + await page + .getByRole('button', { + name: 'Choose local build inputs', + exact: true, + }) + .click(); + const review = page.getByRole('region', { + name: 'Review native build', + }); + await review.waitFor(); + await review + .getByRole('button', { name: 'Build candidate', exact: true }) + .click(); + await page + .getByText('Files verified · Launch untested', { exact: true }) + .waitFor(); + const jobs = await page.evaluate(() => window.loadstone.nativeBuilds()); + assert.equal(jobs[0].status, 'verified'); + for (const output of f.recipe.outputs) { + const file = path.join(jobs[0].destination, output.path); + assert.deepEqual(fingerprint(fs.readFileSync(file)), { + bytes: output.bytes, + sha256: output.sha256, + }); + assert.equal(Math.round(fs.statSync(file).mtimeMs), output.mtimeMs); + } + assert.deepEqual(fs.readdirSync(f.staging), []); + assert.equal( + await page.evaluate( + async () => (await window.loadstone.liveJobs()).length, + ), + 0, + ); + } finally { + await app.close(); + } + }, + ); diff --git a/tests/native-bsa-fixture.mjs b/tests/native-bsa-fixture.mjs new file mode 100644 index 0000000..8787997 --- /dev/null +++ b/tests/native-bsa-fixture.mjs @@ -0,0 +1,73 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { bsaFixture } from './bsa-fixture.mjs'; +import { + createArchiveFixture, + fingerprint, +} from './native-archive-fixture.mjs'; +import { packNativeRecipe } from '../scripts/native-package.mjs'; + +export async function createNativeBsaFixture( + kind = 'zip-windows', + options = {}, +) { + const content = Buffer.from('synthetic mesh'.repeat(1000)); + const files = [ + { path: 'Meshes/first.nif', data: content, compressed: false }, + { path: 'Meshes/shared.nif', data: content, alias: 0, compressed: false }, + { + path: 'Textures/example.dds', + data: Buffer.alloc(200000, 71), + compressed: true, + }, + { path: 'empty.bin', data: Buffer.alloc(0), compressed: false }, + ]; + const encoded = bsaFixture(files, 0x107); + options.mutateBsa?.(encoded); + const second = bsaFixture( + [{ path: 'other.bin', data: 'second container' }], + 7, + ); + const containers = [ + ['Data/Archive A.bsa', encoded.bytes], + ['Data/Archive B.bsa', second.bytes], + ]; + const f = await createArchiveFixture(kind, { + extraFiles: [...containers, ...(options.extraFiles || [])], + timestamps: true, + }); + f.bsaOutputs = []; + for (const [index, [container, bytes]] of containers.entries()) { + for (const file of index + ? [{ path: 'other.bin', data: Buffer.from('second container') }] + : files) { + const output = { + path: `mods/BSA${index}/${file.path}`, + ...fingerprint(Buffer.from(file.data)), + mtimeMs: 946684800321, + action: { + type: 'extract-bsa104', + source: 'original-archive', + container: { member: container, ...fingerprint(bytes) }, + member: file.path, + }, + }; + f.recipe.outputs.push(output); + f.bsaOutputs.push(output); + } + } + const duplicate = { + ...f.bsaOutputs[0], + path: 'mods/BSA0/duplicate.nif', + mtimeMs: 946684801234, + }; + f.recipe.outputs.push(duplicate); + f.bsaOutputs.push(duplicate); + f.packageFile = path.join(f.packages, 'bsa-fixture.loadstone'); + options.mutateRecipe?.(f.recipe); + await packNativeRecipe(f.recipe, f.payloads, f.packageFile); + f.selected.packageFile = f.packageFile; + f.staging = path.join(f.root, 'staging'); + fs.mkdirSync(f.staging); + return f; +} diff --git a/tests/native-bsa.test.mjs b/tests/native-bsa.test.mjs new file mode 100644 index 0000000..59fac51 --- /dev/null +++ b/tests/native-bsa.test.mjs @@ -0,0 +1,359 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import crypto from 'node:crypto'; +import { createRequire } from 'node:module'; +import { createNativeBsaFixture } from './native-bsa-fixture.mjs'; +import { fingerprint } from './native-archive-fixture.mjs'; +const require = createRequire(import.meta.url); +const { materializeNative } = require('../electron/native-materializer.cjs'); +const { inspectNativePackage } = require('../electron/native-package.cjs'); +const { validateRecipe } = require('../electron/native-format.cjs'); +const { + reviewBuildRecovery, +} = require('../electron/native-build-recovery.cjs'); +const { reviewNativeRepair } = require('../electron/native-repair.cjs'); +const { createSevenZip } = require('../electron/sevenzip.cjs'); +function isolate(t, f) { + t.mock.method(os, 'tmpdir', () => f.staging); +} +function verify(f) { + for (const output of f.recipe.outputs) { + const file = path.join(f.destination, output.path); + assert.deepEqual(fingerprint(fs.readFileSync(file)), { + bytes: output.bytes, + sha256: output.sha256, + }); + assert.equal(Math.round(fs.statSync(file).mtimeMs), output.mtimeMs); + } + assert.deepEqual(fs.readdirSync(f.staging), []); + assert.equal( + fs.existsSync(path.join(f.destination, 'Data/Archive A.bsa')), + false, + ); +} +for (const kind of ['zip-windows', '7z', 'rar5']) + test( + `native ${kind} to BSA104 reconstruction verifies exact outputs and timestamps and cleans staging`, + { skip: kind !== 'zip-windows' && !process.env.LOADSTONE_TEST_7ZIP }, + async (t) => { + const f = await createNativeBsaFixture(kind); + isolate(t, f); + const before = fs.readFileSync(f.archive), + stat = fs.statSync(f.archive); + const sevenZip = + kind === 'zip-windows' + ? undefined + : await createSevenZip(process.env.LOADSTONE_TEST_7ZIP); + const result = await materializeNative(f.packageFile, f.destination, { + ...f.options, + sevenZip, + }); + assert.equal(result.status, 'verified'); + verify(f); + assert.deepEqual(fs.readFileSync(f.archive), before); + assert.equal(fs.statSync(f.archive).mtimeMs, stat.mtimeMs); + }, + ); + +test('BSA recipes reject implicit containers, unsafe names, unsupported sources, conflicts and staging budgets', async () => { + const f = await createNativeBsaFixture(); + for (const mutate of [ + (v) => delete v.outputs.at(-1).action.container, + (v) => (v.outputs.at(-1).action.container.member = '../outside.bsa'), + (v) => (v.outputs.at(-1).action.member = 'Meshes\\first.nif'), + (v) => (v.outputs.at(-1).action.source = 'owned-game'), + (v) => (v.outputs.at(-1).action.type = 'extract-bsa105'), + (v) => v.outputs.at(-1).action.container.bytes++, + (v) => (v.outputs.at(-1).action.container.sha256 = '0'.repeat(64)), + (v) => (v.outputs.at(-1).action.container.member = 'data/archive a.bsa'), + (v) => (v.outputs.at(-1).bytes = 1_000_000_001), + (v) => { + for (const output of v.outputs.filter( + (o) => o.action.type === 'extract-bsa104', + )) + output.action.container.bytes = 0x100000000; + }, + (v) => { + for (let i = 0; i < 65; i++) + v.outputs.push({ + ...structuredClone(v.outputs.at(-1)), + path: `mods/extra${i}.bin`, + action: { + ...structuredClone(v.outputs.at(-1).action), + container: { + member: `${i}.bsa`, + bytes: 36, + sha256: '1'.repeat(64), + }, + }, + }); + }, + (v) => { + for (let i = 0; i < 6; i++) + v.outputs.push({ + ...structuredClone(v.outputs.at(-1)), + path: `mods/extra${i}.bin`, + action: { + ...structuredClone(v.outputs.at(-1).action), + container: { + member: `${i}.bsa`, + bytes: 4_000_000_000, + sha256: '1'.repeat(64), + }, + }, + }); + }, + ]) { + const recipe = structuredClone(f.recipe); + mutate(recipe); + assert.throws(() => validateRecipe(recipe)); + } +}); + +test('nested BSA preflight rejects bad containers, members and unrelated unsafe entries before destination writes', async (t) => { + const cases = [ + { + name: 'wrong container hash', + mutateRecipe: (r) => { + for (const o of r.outputs.filter( + (o) => o.action.type === 'extract-bsa104', + )) + o.action.container.sha256 = '0'.repeat(64); + }, + }, + { + name: 'missing container', + mutateRecipe: (r) => { + for (const o of r.outputs.filter( + (o) => o.action.type === 'extract-bsa104', + )) + o.action.container.member = 'Missing/' + o.action.container.member; + }, + }, + { + name: 'wrong inner hash', + mutateRecipe: (r) => { + r.outputs.find((o) => o.action.type === 'extract-bsa104').sha256 = + '0'.repeat(64); + }, + }, + { + name: 'missing inner member', + mutateRecipe: (r) => { + r.outputs.find( + (o) => o.action.type === 'extract-bsa104', + ).action.member = 'missing.nif'; + }, + }, + { + name: 'unsupported BSA version', + mutateBsa: (b) => b.bytes.writeUInt32LE(105, 4), + }, + { + name: 'overlapping BSA ranges', + mutateBsa: (b) => + b.bytes.writeUInt32LE(b.entries[0].offset + 1, b.records[1] + 12), + }, + { + name: 'unsafe outer entry', + extraFiles: [['../unsafe.bin', Buffer.from('unsafe')]], + }, + { + name: 'ambiguous outer entry', + extraFiles: [['data/archive a.bsa', Buffer.from('collision')]], + }, + ]; + for (const item of cases) + await t.test(item.name, async (t) => { + const f = await createNativeBsaFixture('zip-windows', item); + isolate(t, f); + await assert.rejects( + materializeNative(f.packageFile, f.destination, f.options), + ); + assert.equal(fs.existsSync(f.destination), false); + assert.deepEqual(fs.readdirSync(f.staging), []); + }); +}); + +test('BSA staging cancellation and write failures remove temporary files without creating a candidate', async (t) => { + await t.test('cancellation', async (t) => { + const f = await createNativeBsaFixture(); + isolate(t, f); + const controller = new AbortController(); + await assert.rejects( + materializeNative(f.packageFile, f.destination, { + ...f.options, + signal: controller.signal, + activity: (v) => { + if (v.phase === 'archives' && v.totalBytes > 0) controller.abort(); + }, + }), + ); + assert.equal(fs.existsSync(f.destination), false); + assert.deepEqual(fs.readdirSync(f.staging), []); + }); + await t.test('failed disk write', async (t) => { + const f = await createNativeBsaFixture(); + isolate(t, f); + const open = fs.promises.open; + t.mock.method(fs.promises, 'open', async (...args) => { + const handle = await open(...args); + if (String(args[0]).startsWith(f.staging)) + handle.write = async () => { + throw Error('simulated staging write failure'); + }; + return handle; + }); + await assert.rejects( + materializeNative(f.packageFile, f.destination, f.options), + /staging write failure/, + ); + assert.equal(fs.existsSync(f.destination), false); + assert.deepEqual(fs.readdirSync(f.staging), []); + }); + await t.test('insufficient temporary space', async (t) => { + const f = await createNativeBsaFixture(); + isolate(t, f); + const statfs = fs.statfsSync; + t.mock.method(fs, 'statfsSync', (file, options) => + String(file) === f.staging + ? { bavail: 0n, bsize: 4096n } + : statfs(file, options), + ); + await assert.rejects( + materializeNative(f.packageFile, f.destination, f.options), + /temporary disk space/, + ); + assert.equal(fs.existsSync(f.destination), false); + assert.deepEqual(fs.readdirSync(f.staging), []); + }); +}); + +test('BSA cleanup preserves unfamiliar files and refuses a replaced staging directory', async (t) => { + await t.test('unfamiliar file', async (t) => { + const f = await createNativeBsaFixture(); + isolate(t, f); + let canary; + await assert.rejects( + materializeNative(f.packageFile, f.destination, { + ...f.options, + progress: (v) => { + if (v.completed !== 1) return; + const root = path.join(f.staging, fs.readdirSync(f.staging)[0]); + canary = path.join(root, 'unrelated.txt'); + fs.writeFileSync(canary, 'preserve me'); + }, + }), + ); + assert.equal(fs.readFileSync(canary, 'utf8'), 'preserve me'); + assert.deepEqual(fs.readdirSync(path.dirname(canary)), ['unrelated.txt']); + }); + await t.test('replaced directory', async (t) => { + const f = await createNativeBsaFixture(); + isolate(t, f); + let canary, retained; + await assert.rejects( + materializeNative(f.packageFile, f.destination, { + ...f.options, + progress: (v) => { + if (v.completed !== 1) return; + const root = path.join(f.staging, fs.readdirSync(f.staging)[0]); + retained = path.join(f.staging, 'retained-staging'); + assert.equal(path.dirname(root), f.staging); + assert.equal(path.dirname(retained), f.staging); + fs.renameSync(root, retained); + fs.mkdirSync(root); + canary = path.join(root, 'unrelated.txt'); + fs.writeFileSync(canary, 'preserve me'); + }, + }), + /staging directory changed/, + ); + assert.equal(fs.readFileSync(canary, 'utf8'), 'preserve me'); + assert.equal(fs.readdirSync(retained).length, 2); + assert.equal( + JSON.parse( + fs.readFileSync(path.join(f.destination, '.loadstone-native.json')), + ).status, + 'failed', + ); + }); +}); + +test('BSA cancellation resumes exact files and repair retains conflicting files and personal data', async (t) => { + const f = await createNativeBsaFixture(); + isolate(t, f); + const controller = new AbortController(); + await assert.rejects( + materializeNative(f.packageFile, f.destination, { + ...f.options, + signal: controller.signal, + progress: (v) => { + if (v.completed === 3) controller.abort(); + }, + }), + { name: 'AbortError' }, + ); + assert.deepEqual(fs.readdirSync(f.staging), []); + assert.equal( + JSON.parse( + fs.readFileSync(path.join(f.destination, '.loadstone-native.json')), + ).status, + 'cancelled', + ); + const inspected = await inspectNativePackage(f.packageFile); + const reviewed = await reviewBuildRecovery(inspected, f.destination); + assert.ok(reviewed.reused > 0); + await materializeNative(f.packageFile, f.destination, { + ...f.options, + resume: reviewed, + }); + verify(f); + const changed = f.bsaOutputs[0], + missing = f.bsaOutputs[2]; + const kept = fs.statSync(path.join(f.destination, f.bsaOutputs[1].path)); + fs.writeFileSync(path.join(f.destination, changed.path), 'personal mesh'); + fs.unlinkSync(path.join(f.destination, missing.path)); + fs.mkdirSync(path.join(f.destination, 'profiles/Test/saves')); + fs.writeFileSync( + path.join(f.destination, 'profiles/Test/saves/test.fos'), + 'save canary', + ); + fs.writeFileSync( + path.join(f.destination, 'untracked.bin'), + 'untracked canary', + ); + const repair = await reviewNativeRepair(inspected, f.destination), + directory = `.loadstone-repair-recovery/${crypto.randomUUID()}`; + assert.equal(repair.conflicts, 1); + assert.equal(repair.missing, 1); + const result = await materializeNative(f.packageFile, f.destination, { + ...f.options, + resume: repair, + repair: { directory }, + }); + assert.equal(result.status, 'repaired'); + verify(f); + assert.equal( + fs.readFileSync(path.join(f.destination, directory, changed.path), 'utf8'), + 'personal mesh', + ); + assert.equal( + fs.readFileSync( + path.join(f.destination, 'profiles/Test/saves/test.fos'), + 'utf8', + ), + 'save canary', + ); + assert.equal( + fs.readFileSync(path.join(f.destination, 'untracked.bin'), 'utf8'), + 'untracked canary', + ); + assert.equal( + fs.statSync(path.join(f.destination, f.bsaOutputs[1].path)).ino, + kept.ino, + ); +});