diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29dd7ce..ed55b7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,10 @@ jobs: cmp docs/demo/git-pack-index-v1/verify.txt "$RUNNER_TEMP/pack-verify.txt" env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab pack-inspect --compact > "$RUNNER_TEMP/pack-inspect.json" cmp evidence/git-pack-index-v1.json "$RUNNER_TEMP/pack-inspect.json" + env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab pack-ofs-verify > "$RUNNER_TEMP/pack-ofs-verify.txt" + cmp docs/demo/git-pack-ofs-delta-v1/verify.txt "$RUNNER_TEMP/pack-ofs-verify.txt" + env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab pack-ofs-inspect --compact > "$RUNNER_TEMP/pack-ofs-inspect.json" + cmp evidence/git-pack-ofs-delta-v1.json "$RUNNER_TEMP/pack-ofs-inspect.json" - name: Check tests, evidence freshness, and shell boundary shell: bash @@ -76,10 +80,12 @@ jobs: env -u PYTHONHOME -u PYTHONPATH python -W error -m unittest discover -s tests -v env -u PYTHONHOME -u PYTHONPATH python -B tools/generate_evidence.py --check env -u PYTHONHOME -u PYTHONPATH python -B tools/generate_pack_evidence.py --check + env -u PYTHONHOME -u PYTHONPATH python -B tools/generate_ofs_evidence.py --check bash -n tools/capture_report.sh bash -n tools/capture_pack_report.sh + bash -n tools/capture_ofs_report.sh test "$(shellcheck --version | awk '$1 == "version:" {print $2}')" = "0.9.0" - shellcheck tools/capture_report.sh tools/capture_pack_report.sh + shellcheck tools/capture_report.sh tools/capture_pack_report.sh tools/capture_ofs_report.sh test "$(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none)" = "" distribution: @@ -130,6 +136,11 @@ jobs: env -u PYTHONHOME -u PYTHONPATH "$runtime/bin/python" -B -m git_dag_lab pack-verify ) > "$RUNNER_TEMP/wheel-pack-verify.txt" cmp docs/demo/git-pack-index-v1/verify.txt "$RUNNER_TEMP/wheel-pack-verify.txt" + ( + cd "$RUNNER_TEMP" + env -u PYTHONHOME -u PYTHONPATH "$runtime/bin/python" -B -m git_dag_lab pack-ofs-verify + ) > "$RUNNER_TEMP/wheel-pack-ofs-verify.txt" + cmp docs/demo/git-pack-ofs-delta-v1/verify.txt "$RUNNER_TEMP/wheel-pack-ofs-verify.txt" test "$(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none)" = "" - name: Upload verified distributions diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 430e0eb..739a0f1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,12 +35,12 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@c4dd10e44af883a891fe31ced449bcb4a6728b9b # v3.37.6 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} queries: security-extended - name: Analyze - uses: github/codeql-action/analyze@c4dd10e44af883a891fe31ced449bcb4a6728b9b # v3.37.6 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: category: /language:${{ matrix.language }} diff --git a/README.md b/README.md index 9696591..b8b9c03 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,8 @@ python3 -m git_dag_lab verify python3 -m git_dag_lab inspect python3 -m git_dag_lab pack-verify python3 -m git_dag_lab pack-inspect +python3 -m git_dag_lab pack-ofs-verify +python3 -m git_dag_lab pack-ofs-inspect python3 -m unittest discover -s tests -v ``` @@ -59,7 +61,21 @@ The verifier does not trust the pack filename or Git's index. It parses the vari ![Receipt-derived integrity chain from pack header through index checksum](docs/assets/git-pack-integrity.svg) -This is deliberately a closed subset: pack v2 and index v2, at most 64 objects, 1 MiB files, 256 KiB expanded objects, and non-delta entries only. Delta entries (OFS/REF), other object formats, arbitrary repositories, reachability, and pack optimization are not claimed. SHA-1 and CRC32 model Git storage integrity here; neither is presented as authentication, a signature, or collision-resistant security. +The checked-in production pack evidence remains a deliberately closed non-delta baseline: pack v2 and index v2, at most 64 objects, 1 MiB files, and 256 KiB expanded objects. The parser accepts bounded OFS_DELTA entries with exact earlier-entry bases, depth 4, 4,096 instructions, three-byte offset/size ceilings, and a 16 MiB aggregate expansion budget. REF_DELTA, thin packs, other object formats, arbitrary repositories, reachability, and caller-selected pack optimization remain explicitly unsupported. SHA-1 and CRC32 model Git storage integrity here; neither is presented as authentication, a signature, or collision-resistant security. + +### Delta runtime: real OFS bytes + +The separate `pack-ofs-*` path stores two fixed 77,824-byte synthetic blobs, changes exactly one numbered record, and asks real `git pack-objects --delta-base-offset` for a depth-one pack. It fails closed unless Git emits exactly one full blob plus one OFS_DELTA, then independently reconstructs both logical objects and cross-checks their physical offsets and CRC32 rows against index v2. Its normalized argv and exact stdin digest are receipt-bound; byte identity is claimed only for repeated runs with the same recorded Git build. + +![Actual pinned-Chromium screenshot of the real OFS_DELTA report, physical entries, checks, command provenance, and CLI receipt](docs/assets/git-pack-ofs-report.png) + +

Attested 1440×1500 Chromium capture. Provenance: OFS report receipt `8c2b8b07…540b0`; screenshot SHA-256 `2e3a4e81…a3041`. Git 2.54.0 generated one full blob and one OFS_DELTA from two fixed synthetic blobs; zero REF_DELTA, thin-pack, arbitrary-repository, authentication, network, secret, or host-data claims.

+ +![Actual physical OFS base offset, biased distance bytes, independent reconstruction, logical OIDs, index offsets, and CRC32 rows](docs/assets/git-pack-ofs-reconstruction.svg) + +![Source-bound workflow from fixed inputs and exact Git argv through independent replay, index binding, and receipt](docs/assets/git-pack-ofs-workflow.svg) + +![Exact real pack-ofs-verify stdout with actual pack and receipt digests](docs/assets/git-pack-ofs-cli.svg) ## The hard part: verify Git without trusting Git @@ -88,26 +104,28 @@ See [SECURITY.md](SECURITY.md) for the threat model and trusted-input boundary. ## Evidence pipeline -Every README visual begins with a canonical production CLI document. The DAG and pack generators each run fresh experiments twice, require byte-identical outputs, derive their SVGs and offline HTML, and bind every artifact into a hash manifest. Digest-pinned Chromium captures both reports in read-only containers with `--network none`. Separate attestations bind each exact report, rendered DOM, PNG, browser binary/version, container digest, isolation policy, viewport, and capture-script hash; without the matching attestation, a generator refuses to call its screenshot verified. +Every README visual begins with a canonical production CLI document. The DAG, non-delta pack, and OFS generators each run fresh experiments twice, require byte-identical outputs under their recorded runtime, derive their SVGs and offline HTML, and bind every artifact into a hash manifest. Digest-pinned Chromium captures all three reports in read-only containers with `--network none`. Separate attestations bind each exact report, rendered DOM, PNG, browser binary/version, container digest, isolation policy, viewport, and capture-script hash; without the matching attestation, a generator refuses to call its screenshot verified. ![Architecture of the fixed scenario, real Git plumbing, independent verification, and evidence publication pipeline](docs/assets/evidence-pipeline.svg) ### Reproduce the checked-in evidence ```bash -# Verify both JSON/transcript/visual/report/manifest packages and PNG attestations. +# Verify all JSON/transcript/visual/report/manifest packages and PNG attestations. python3 -B tools/generate_evidence.py --check python3 -B tools/generate_pack_evidence.py --check +python3 -B tools/generate_ofs_evidence.py --check -# Rebuild and recapture either offline report with pinned Chromium. +# Rebuild and recapture any offline report with pinned Chromium. tools/capture_report.sh tools/capture_pack_report.sh +tools/capture_ofs_report.sh # Run all parser, boundary, CLI, evidence, and provenance tests. python3 -W error -m unittest discover -s tests -v ``` -Current verified baseline: **90 tests**, **9/9 graph invariants**, **7/7 pack/index checks**, **57 isolated Git invocations** in the DAG evidence run, two independently replayed evidence packages, and two attested offline browser captures. +Current verified baseline: **113 tests**, **9/9 graph invariants**, **7/7 baseline pack checks**, **12/12 OFS checks**, **57 isolated Git invocations** in the DAG evidence run, three independently replayed evidence packages, and three attested offline browser captures. | Artifact | What it proves | |---|---| @@ -126,13 +144,19 @@ Current verified baseline: **90 tests**, **9/9 graph invariants**, **7/7 pack/in | [`git-pack-integrity.svg`](docs/assets/git-pack-integrity.svg) | Receipt-derived pack/index checksum and row-binding workflow | | [`git-pack-report.png`](docs/assets/git-pack-report.png) | Actual Chromium rendering of the pack/index report at 1440×1500 | | [`git-pack-index-v1/manifest.json`](docs/demo/git-pack-index-v1/manifest.json) | Hash/size/source/command/capture inventory for every pack visual and output | +| [`evidence/git-pack-ofs-delta-v1.json`](evidence/git-pack-ofs-delta-v1.json) | Canonical real OFS_DELTA receipt, fixed mutation, physical representation, reconstructed logical objects, and non-claims | +| [`git-pack-ofs-cli.svg`](docs/assets/git-pack-ofs-cli.svg) | Exact production `pack-ofs-verify` stdout rendered as an accessible terminal panel | +| [`git-pack-ofs-reconstruction.svg`](docs/assets/git-pack-ofs-reconstruction.svg) | Actual OFS distance bytes, base edge, independent reconstruction, OIDs, index offsets, and CRC32 | +| [`git-pack-ofs-workflow.svg`](docs/assets/git-pack-ofs-workflow.svg) | Fixed inputs, normalized Git argv, recorded Git build, parser, index binding, and receipt | +| [`git-pack-ofs-report.png`](docs/assets/git-pack-ofs-report.png) | Actual Chromium rendering of the OFS_DELTA report at 1440×1500 | +| [`git-pack-ofs-delta-v1/manifest.json`](docs/demo/git-pack-ofs-delta-v1/manifest.json) | Hash/size/source/Git-build/command/capture inventory for every OFS visual and output | ## Test coverage by risk The standard-library suite exercises more than happy-path graph construction: - independent blob, tree, and commit envelope hashes; -- pack v2 headers, bounded zlib streams, logical OIDs, trailer checksum, and explicit delta rejection; +- pack v2 headers, bounded zlib streams, logical OIDs, trailer checksum, bounded OFS_DELTA replay, and explicit REF_DELTA rejection; - index v2 fanout, sorted OIDs, CRC32 rows, small/large offsets, pack binding, and checksum mutations; - exact object/ref inventories, parent ordering, reachability, and ancestry; - Git's special `directory/` tree ordering, truncated binary objects, and malformed headers; diff --git a/SECURITY.md b/SECURITY.md index c7474f8..0beebbb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -19,9 +19,9 @@ Evidence refreshes reject symlinked targets and multi-link files, stage each gen ## Pack/index closed subset -The pack experiment passes only three fixed synthetic blob IDs to `git pack-objects`; it does not accept a repository path, revision, ref, or caller-provided object list. Generated `.pack` and `.idx` files must be regular, single-link files no larger than 1 MiB and must remain the same inode and size across the bounded read. +The baseline pack experiment passes only three fixed synthetic blob IDs to `git pack-objects`. The separate OFS experiment passes two fixed 77,824-byte synthetic blob IDs and fixed single-threaded delta options. Neither accepts a repository path, revision, ref, caller-provided object list, or caller-selected optimizer settings. Generated `.pack` and `.idx` files must be regular, single-link files no larger than 1 MiB and must remain the same inode and size across the bounded read. -The independent parser accepts pack v2 and index v2 only. It rejects OFS/REF deltas, more than 64 objects, objects expanding beyond 256 KiB, invalid or unterminated zlib streams, duplicate logical objects, non-canonical fanout/large-offset tables, and any mismatch among logical object IDs, CRC32 rows, offsets, pack trailer, index pack binding, or index checksum. These checks establish the fixed fixture's storage integrity; they do not establish provenance, authenticity, repository reachability, or safety of arbitrary Git data. +The independent parser accepts pack v2 and index v2 with logical commit, tree, and blob entries only; tag entries are outside this closed subset. Its bounded OFS_DELTA subset requires an exact earlier pack-entry base, canonical biased offsets and size headers, no more than depth 4 or 4,096 instructions, at most three bytes for bounded offsets and sizes, at most 256 KiB per object or delta program, and at most 16 MiB of aggregate expanded output. It rejects REF_DELTA and thin packs, more than 64 objects, invalid or unterminated zlib streams, duplicate logical objects, non-canonical fanout/large-offset tables, and any mismatch among logical object IDs, CRC32 rows, offsets, pack trailer, index pack binding, or index checksum. The OFS runtime additionally requires exactly one full blob, one depth-one OFS entry, an exact re-encoding of the observed biased distance, and a complete fixed logical inventory. Its report records normalized pack arguments and an exact stdin digest; output bytes are only expected to repeat under the same Git build. These checks establish bounded fixture storage integrity; they do not establish provenance, authenticity, repository reachability, collision resistance, or safety of arbitrary Git data. ## SHA-1 scope diff --git a/docs/assets/git-pack-ofs-cli.svg b/docs/assets/git-pack-ofs-cli.svg new file mode 100644 index 0000000..a67e0cf --- /dev/null +++ b/docs/assets/git-pack-ofs-cli.svg @@ -0,0 +1,29 @@ + + Real OFS_DELTA CLI receipt + Exact stdout from pack-ofs-verify after real git pack-objects output was reconstructed and cross-checked against index v2. + {"report_receipt_sha256":"8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0","source":"evidence/git-pack-ofs-delta-v1.json"} + + + + + + + + REAL CLI STDOUT · STDERR 0 BYTES · EXIT 0 + + + $ python3 -B -m git_dag_lab pack-ofs-verify + PASS git-pack-ofs-delta-lab/v1 objects=2 pack_version=2 index_version=2 + deltas=1 pack_sha1=736fdd73b58a6a68dfefec9288533a8d0a41a9e6 + receipt_sha256=8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0 + receipt-bound source · evidence/git-pack-ofs-delta-v1.json + receipt 8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0 + diff --git a/docs/assets/git-pack-ofs-reconstruction.svg b/docs/assets/git-pack-ofs-reconstruction.svg new file mode 100644 index 0000000..1bc4177 --- /dev/null +++ b/docs/assets/git-pack-ofs-reconstruction.svg @@ -0,0 +1,41 @@ + + Real OFS_DELTA reconstruction + Physical offsets, the biased backward distance, logical object sizes, and reconstructed object IDs come from the verified Git-generated pack. + {"base_oid":"586c379527cbf256bb8dc7558e808ae5150a8959","delta_oid":"d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6","pack_sha256":"080b1ca0b5cdb6c5e4cac746a48b8feb218aa7ec0b5583024a493d110c319a31","report_receipt_sha256":"8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0","source":"evidence/git-pack-ofs-delta-v1.json"} + + + + + + + + REAL GIT PACK · ONE FULL BLOB · ONE OFS_DELTA · ZERO REF_DELTA + A physical backward edge reconstructs a new logical blob. + + BASE ENTRY · FULL + offset 12 + 77843 packed bytes · 77824 logical bytes + line-1024-changed + 586c379527cbf256bb8dc7558e808ae5150a8959 + + DELTA ENTRY · TYPE 6 + offset 77855 − distance 77843 + biased bytes 0x83df13 · program 26 bytes + baseline · depth 1 + d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6 + + base_offset = 12 + + INDEPENDENT REPLAY · 12/12 CHECKS + Base OID 586c379527cbf256bb8dc7558e808ae5150a8959 → apply bounded copy/insert program → target OID d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6 + Index v2 binds physical offsets 12 and 77855, plus CRC32 b6dfc85b and 84af9f53. + receipt 8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0 + diff --git a/docs/assets/git-pack-ofs-report.png b/docs/assets/git-pack-ofs-report.png new file mode 100644 index 0000000..ff7fe72 Binary files /dev/null and b/docs/assets/git-pack-ofs-report.png differ diff --git a/docs/assets/git-pack-ofs-workflow.svg b/docs/assets/git-pack-ofs-workflow.svg new file mode 100644 index 0000000..da6bbff --- /dev/null +++ b/docs/assets/git-pack-ofs-workflow.svg @@ -0,0 +1,56 @@ + + Reproducible OFS evidence workflow + The exact fixed inputs, normalized Git arguments, independent parser, index cross-check, and receipt shown here are bound into the evidence manifest. + {"git_frontend_sha256":"f54a87f6253aab09ed7b522bd78ddeab509105b1043076209d89127e55877a48","report_receipt_sha256":"8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0","source":"evidence/git-pack-ofs-delta-v1.json","stdin_sha256":"fa3a61ef14c4439502fdbfbaf0fe8a5b666b5ee54f70d48b99a14beb172cf952"} + + + + + + + + FIXED INPUTS · SINGLE THREAD · SAME RECORDED GIT BUILD · FRESH RUNS ×2 + From two synthetic blobs to one source-bound receipt. + + 01 + FIXTURE + 2 × 77,824 B + PASS + + + 02 + REAL GIT + git version 2.54.0 + PASS + + + 03 + PACK PARSER + OFS replay + PASS + + + 04 + INDEX V2 + offset + CRC + PASS + + + 05 + RECEIPT + 8c2b8b07e9ab4849… + PASS + + $ git pack-objects --delta-base-offset --window=2 --depth=1 --threads=1 + --compression=0 --no-reuse-delta --no-reuse-object --index-version=2 <private>/fixture + stdin SHA-256 fa3a61ef14c4439502fdbfbaf0fe8a5b666b5ee54f70d48b99a14beb172cf952 + Git frontend SHA-256 f54a87f6253aab09ed7b522bd78ddeab509105b1043076209d89127e55877a48 · byte identity scope: same recorded build + diff --git a/docs/demo/git-dag-v1/manifest.json b/docs/demo/git-dag-v1/manifest.json index d8492cf..defb3d9 100644 --- a/docs/demo/git-dag-v1/manifest.json +++ b/docs/demo/git-dag-v1/manifest.json @@ -135,8 +135,8 @@ }, { "path": "git_dag_lab/cli.py", - "sha256": "03648331670cdc350fd486b2939e4914e90ee7ca7c2ea2b5cabd4eced21fe766", - "size": 2335 + "sha256": "e634feb9a908e3b0f46e1011c64667000b9e43ceb6ce1233b9fda1ce166a35ac", + "size": 2972 }, { "path": "tools/generate_evidence.py", diff --git a/docs/demo/git-pack-index-v1/manifest.json b/docs/demo/git-pack-index-v1/manifest.json index 60ec3f1..c61735c 100644 --- a/docs/demo/git-pack-index-v1/manifest.json +++ b/docs/demo/git-pack-index-v1/manifest.json @@ -130,13 +130,13 @@ "sources": [ { "path": "git_dag_lab/pack.py", - "sha256": "d3882af57ad9b3d85580032b7e5ceccd2c8b443fbee7be30c7452a31aee3f8a8", - "size": 20802 + "sha256": "b2b7061751f09997673b09a235e5257c8a2e93de82ad523703aa1bc015d4c765", + "size": 39441 }, { "path": "git_dag_lab/cli.py", - "sha256": "03648331670cdc350fd486b2939e4914e90ee7ca7c2ea2b5cabd4eced21fe766", - "size": 2335 + "sha256": "e634feb9a908e3b0f46e1011c64667000b9e43ceb6ce1233b9fda1ce166a35ac", + "size": 2972 }, { "path": "tools/generate_evidence.py", diff --git a/docs/demo/git-pack-ofs-delta-v1/capture-attestation.json b/docs/demo/git-pack-ofs-delta-v1/capture-attestation.json new file mode 100644 index 0000000..ed5a6fe --- /dev/null +++ b/docs/demo/git-pack-ofs-delta-v1/capture-attestation.json @@ -0,0 +1,60 @@ +{ + "attestation": { + "browser": { + "binary_path": "/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell", + "sha256": "003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d", + "version": "Chromium 140.0.7339.186" + }, + "container": { + "architecture": "amd64", + "image": "mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c" + }, + "input": { + "report": { + "path": "docs/demo/git-pack-ofs-delta-v1/report.html", + "report_receipt_sha256": "8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0", + "sha256": "ac67aa531f616b148df847a92e3a228063ea9591338054b247118f4065d7b486", + "size": 6288 + } + }, + "isolation": { + "capabilities": "all-dropped", + "demo_mount": "read-only", + "network": "none", + "no_new_privileges": true, + "pull": "never", + "root_filesystem": "read-only", + "user": "caller-nonroot" + }, + "outputs": { + "rendered_dom": { + "path": "docs/demo/git-pack-ofs-delta-v1/rendered-dom.html", + "sha256": "acf985e2f5a5d68ef02ecad57dcbf9493f8bcd5b2aa36dfac1a39dd5b748e0d5", + "size": 6288 + }, + "screenshot": { + "height": 1500, + "path": "docs/assets/git-pack-ofs-report.png", + "sha256": "2e3a4e813cebf28a18d60bc65d9d3eb5d5426b2e702ac36a560a7c52465a3041", + "size": 598564, + "width": 1440 + } + }, + "schema_version": "git-pack-ofs-browser-capture-attestation/v1", + "script": { + "path": "tools/capture_ofs_report.sh", + "sha256": "638de7acced052805c52e2bb85de3dd1697a61254595299111f8ab6fea3afae1", + "size": 11144 + }, + "viewport": { + "device_scale_factor": 1, + "height": 1500, + "width": 1440 + } + }, + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": "5891c29f10f3f5bddd736aae13829a16c4899a6cda2c47952d190025342c31b9" + } +} diff --git a/docs/demo/git-pack-ofs-delta-v1/inspect.json b/docs/demo/git-pack-ofs-delta-v1/inspect.json new file mode 100644 index 0000000..bfc2e5f --- /dev/null +++ b/docs/demo/git-pack-ofs-delta-v1/inspect.json @@ -0,0 +1,151 @@ +{ + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": "8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0" + }, + "report": { + "checks": { + "all_fixture_objects_present": true, + "exactly_one_full_entry": true, + "exactly_one_ofs_delta_entry": true, + "index_checksum_verified": true, + "index_crc32_matches_pack": true, + "index_fanout_matches_sorted_oids": true, + "index_offsets_match_pack": true, + "ofs_base_entry_bound": true, + "ofs_distance_reencoded": true, + "pack_trailer_verified": true, + "reconstructed_objects_match_fixture": true, + "ref_delta_entries_absent": true + }, + "command_trace": [ + "init", + "hash-object", + "hash-object", + "pack-objects" + ], + "fixture": { + "changed_record": { + "after": "1024:fedcba9876543210fedcba9876543210", + "before": "1024:0123456789abcdef0123456789abcdef", + "line_number": 1024 + }, + "object_count": 2, + "objects": [ + { + "label": "baseline", + "oid": "d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6", + "payload_sha256": "ea612ce3c3fb55f7a05ceecf1bcbf0ecf62fe298a8e387011e21f0d3d6353f1a", + "size": 77824 + }, + { + "label": "line-1024-changed", + "oid": "586c379527cbf256bb8dc7558e808ae5150a8959", + "payload_sha256": "e6bd970352e9ed0dc5389db5f9e4965c3d17e937881c69d4e7edf3df92e65dd4", + "size": 77824 + } + ] + }, + "index": { + "bytes": 1128, + "index_sha1": "acc33096abf3927200be57616ce87669d43efeeb", + "nonzero_fanout_buckets": [ + { + "cumulative": 1, + "prefix": "58", + "range_start": 0 + }, + { + "cumulative": 2, + "prefix": "d2", + "range_start": 1 + } + ], + "pack_sha1": "736fdd73b58a6a68dfefec9288533a8d0a41a9e6", + "sha256": "57fb9cfc0ac21a82384477f34b05e575b46f2a344bc10942dd4216603e5cb206", + "version": 2 + }, + "object_format": "sha1", + "objects_in_pack_order": [ + { + "crc32": "b6dfc85b", + "index_crc32": "b6dfc85b", + "index_offset": 12, + "label": "line-1024-changed", + "object_type": "blob", + "offset": 12, + "oid": "586c379527cbf256bb8dc7558e808ae5150a8959", + "packed_size": 77843, + "payload_sha256": "e6bd970352e9ed0dc5389db5f9e4965c3d17e937881c69d4e7edf3df92e65dd4", + "representation": "full", + "size": 77824 + }, + { + "base_label": "line-1024-changed", + "base_offset": 12, + "base_oid": "586c379527cbf256bb8dc7558e808ae5150a8959", + "crc32": "84af9f53", + "delta_depth": 1, + "index_crc32": "84af9f53", + "index_offset": 77855, + "label": "baseline", + "object_type": "blob", + "offset": 77855, + "ofs_distance": 77843, + "ofs_offset_bytes_hex": "83df13", + "oid": "d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6", + "packed_size": 42, + "payload_sha256": "ea612ce3c3fb55f7a05ceecf1bcbf0ecf62fe298a8e387011e21f0d3d6353f1a", + "representation": "ofs-delta", + "size": 77824, + "stored_size": 26 + } + ], + "pack": { + "bytes": 77917, + "delta_count": 1, + "full_count": 1, + "max_delta_depth": 1, + "object_count": 2, + "ofs_delta_count": 1, + "ref_delta_count": 0, + "sha256": "080b1ca0b5cdb6c5e4cac746a48b8feb218aa7ec0b5583024a493d110c319a31", + "trailer_sha1": "736fdd73b58a6a68dfefec9288533a8d0a41a9e6", + "version": 2 + }, + "pack_objects": { + "normalized_argv": [ + "git", + "pack-objects", + "--delta-base-offset", + "--window=2", + "--depth=1", + "--threads=1", + "--compression=0", + "--no-reuse-delta", + "--no-reuse-object", + "--index-version=2", + "/fixture" + ], + "stdin_bytes": 82, + "stdin_oid_order": [ + "586c379527cbf256bb8dc7558e808ae5150a8959", + "d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6" + ], + "stdin_sha256": "fa3a61ef14c4439502fdbfbaf0fe8a5b666b5ee54f70d48b99a14beb172cf952" + }, + "schema_version": "git-pack-ofs-delta-lab/v1", + "scope": { + "arbitrary_repository_supported": false, + "authentication_claim": false, + "byte_identity_requires_same_git_build": true, + "fixture_kind": "two deterministic similar synthetic blobs", + "git_pack_objects_executed": true, + "network_required": false, + "ofs_delta_supported": true, + "ref_delta_supported": false, + "thin_pack_supported": false + } + } +} diff --git a/docs/demo/git-pack-ofs-delta-v1/manifest.json b/docs/demo/git-pack-ofs-delta-v1/manifest.json new file mode 100644 index 0000000..49632e0 --- /dev/null +++ b/docs/demo/git-pack-ofs-delta-v1/manifest.json @@ -0,0 +1,177 @@ +{ + "artifacts": [ + { + "path": "docs/assets/git-pack-ofs-cli.svg", + "role": "visualized exact pack-ofs-verify transcript", + "sha256": "66e4ca96db44e7fbb4e6c31e95385fd963a270854b37a973943d0279bfdd4e8f", + "size": 2768 + }, + { + "path": "docs/assets/git-pack-ofs-reconstruction.svg", + "role": "actual OFS base distance and reconstructed object flow", + "sha256": "e2c655c185b562767e4bd92b7de45be11b49425e0800a291ab24dd104339ba32", + "size": 3884 + }, + { + "path": "docs/assets/git-pack-ofs-report.png", + "role": "attested OFS_DELTA offline report browser capture", + "sha256": "2e3a4e813cebf28a18d60bc65d9d3eb5d5426b2e702ac36a560a7c52465a3041", + "size": 598564 + }, + { + "path": "docs/assets/git-pack-ofs-workflow.svg", + "role": "source-bound Git OFS evidence workflow", + "sha256": "3641349f1632c500ef657c15a6e3f1c53b54b750bddf4c732e7b5d13d9c05610", + "size": 4689 + }, + { + "path": "docs/demo/git-pack-ofs-delta-v1/capture-attestation.json", + "role": "OFS capture provenance and isolation attestation", + "sha256": "f4d4fd7ec4551516c33e452e447dc55b4572950d3faa1fac155985c1c81e0107", + "size": 2026 + }, + { + "path": "docs/demo/git-pack-ofs-delta-v1/inspect.json", + "role": "exact pretty pack-ofs-inspect stdout", + "sha256": "3234380d7f5e9a6590bb3f0ee33206c8cf588cc8af054979448d4020c5421637", + "size": 4712 + }, + { + "path": "docs/demo/git-pack-ofs-delta-v1/rendered-dom.html", + "role": "actual DOM emitted during the OFS report capture", + "sha256": "acf985e2f5a5d68ef02ecad57dcbf9493f8bcd5b2aa36dfac1a39dd5b748e0d5", + "size": 6288 + }, + { + "path": "docs/demo/git-pack-ofs-delta-v1/report.html", + "role": "dependency-free OFS_DELTA offline report", + "sha256": "ac67aa531f616b148df847a92e3a228063ea9591338054b247118f4065d7b486", + "size": 6288 + }, + { + "path": "docs/demo/git-pack-ofs-delta-v1/verify.txt", + "role": "exact pack-ofs-verify stdout", + "sha256": "c33227146c3483414cb444d82ddcffb41abda71a9911d622e0e0f0cc66721d59", + "size": 212 + }, + { + "path": "evidence/git-pack-ofs-delta-v1.json", + "role": "canonical compact real OFS_DELTA CLI evidence", + "sha256": "0a5a820ae6a1d985ff54f99f10b632a548a3cf84f36fceffacc359be62c942e1", + "size": 3459 + } + ], + "capture": { + "attestation": "docs/demo/git-pack-ofs-delta-v1/capture-attestation.json", + "attestation_receipt_sha256": "5891c29f10f3f5bddd736aae13829a16c4899a6cda2c47952d190025342c31b9", + "browser_binary": "/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell", + "browser_sha256": "003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d", + "browser_version": "Chromium 140.0.7339.186", + "container_image": "mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c", + "network": "none", + "rendered_dom_sha256": "acf985e2f5a5d68ef02ecad57dcbf9493f8bcd5b2aa36dfac1a39dd5b748e0d5", + "screenshot_sha256": "2e3a4e813cebf28a18d60bc65d9d3eb5d5426b2e702ac36a560a7c52465a3041", + "status": "attested", + "viewport": { + "height": 1500, + "width": 1440 + } + }, + "commands": [ + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-ofs-verify" + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": "docs/demo/git-pack-ofs-delta-v1/verify.txt" + }, + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-ofs-inspect", + "--compact" + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": "evidence/git-pack-ofs-delta-v1.json" + }, + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-ofs-inspect" + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": "docs/demo/git-pack-ofs-delta-v1/inspect.json" + } + ], + "git_build": { + "byte_identity_scope": "same recorded Git build", + "frontend_sha256": "f54a87f6253aab09ed7b522bd78ddeab509105b1043076209d89127e55877a48", + "pack_objects_sha256": "f54a87f6253aab09ed7b522bd78ddeab509105b1043076209d89127e55877a48", + "version": "git version 2.54.0" + }, + "report_receipt_sha256": "8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0", + "schema_version": "git-pack-ofs-evidence-manifest/v1", + "sources": [ + { + "path": "git_dag_lab/__init__.py", + "sha256": "c3aa1ce7deabf04c718f103bc2ba93cd964efbf5cff159b26d87fbd1c6bd4279", + "size": 475 + }, + { + "path": "git_dag_lab/__main__.py", + "sha256": "33386ce299571769ff97cf7c6757336ff49ed023357bdcd1579d502d667391a5", + "size": 105 + }, + { + "path": "git_dag_lab/cli.py", + "sha256": "e634feb9a908e3b0f46e1011c64667000b9e43ceb6ce1233b9fda1ce166a35ac", + "size": 2972 + }, + { + "path": "git_dag_lab/lab.py", + "sha256": "289e688e84024c20b94f3aa2e59be693b81b507b8581fe5547cc8f5eb223879b", + "size": 34676 + }, + { + "path": "git_dag_lab/pack.py", + "sha256": "b2b7061751f09997673b09a235e5257c8a2e93de82ad523703aa1bc015d4c765", + "size": 39441 + }, + { + "path": "tools/generate_evidence.py", + "sha256": "9276b529ec13eaa6ceda1cab1282b6679a689697e5c6930ca17b5e0bc8ba3a92", + "size": 49590 + }, + { + "path": "tools/generate_pack_evidence.py", + "sha256": "234af1ac72189cec9034d3b7f396f6950b01564495208a8891620bb7721ec3a9", + "size": 41376 + }, + { + "path": "tools/generate_ofs_evidence.py", + "sha256": "866a7fbe41d0cf87386f07d7ee82cf48382836333a2ee7f3cda3862d0ea26c7a", + "size": 41449 + }, + { + "path": "tools/capture_ofs_report.sh", + "sha256": "638de7acced052805c52e2bb85de3dd1697a61254595299111f8ab6fea3afae1", + "size": 11144 + } + ] +} diff --git a/docs/demo/git-pack-ofs-delta-v1/rendered-dom.html b/docs/demo/git-pack-ofs-delta-v1/rendered-dom.html new file mode 100644 index 0000000..9f84ce6 --- /dev/null +++ b/docs/demo/git-pack-ofs-delta-v1/rendered-dom.html @@ -0,0 +1,36 @@ + + + + + + Git OFS_DELTA evidence — verified offline report + + + +
+

Git pack lab · actual OFS_DELTA offline report

+

One backward edge.
Two verified blobs.

+

Real git pack-objects --delta-base-offset output is decoded without asking Git to replay it: the bounded delta program reconstructs the target, then index v2 independently binds both physical offsets and CRC32 rows.

+
PASS · report receipt 8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0
+
+
2logical blobs
+
1physical OFS_DELTA
+
77917actual pack bytes
+
12/12checks passed
+
+
+

Objects in physical pack order

formfixtureOIDoffsetstoredCRC32
fullline-1024-changed586c379527cbf256bb8dc7558e808ae5150a89591277824b6dfc85b
ofs-deltabaselined2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6778552684af9f53
Base → target
offset 12 → 77855; biased distance 77843 encoded as 83df13
586c379527cbf256bb8dc7558e808ae5150a8959
↓ independent copy/insert replay
d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6
+

Independent fail-closed gate

  • ✓all fixture objects present
  • ✓exactly one full entry
  • ✓exactly one ofs delta entry
  • ✓index checksum verified
  • ✓index crc32 matches pack
  • ✓index fanout matches sorted oids
  • ✓index offsets match pack
  • ✓ofs base entry bound
  • ✓ofs distance reencoded
  • ✓pack trailer verified
  • ✓reconstructed objects match fixture
  • ✓ref delta entries absent
+
+

Exact command provenance

$ git pack-objects --delta-base-offset --window=2 --depth=1 --threads=1 --compression=0 --no-reuse-delta --no-reuse-object --index-version=2 <private>/fixture
+stdin sha256=fa3a61ef14c4439502fdbfbaf0fe8a5b666b5ee54f70d48b99a14beb172cf952
+Git=git version 2.54.0
+frontend sha256=f54a87f6253aab09ed7b522bd78ddeab509105b1043076209d89127e55877a48

Real CLI receipt

$ python3 -B -m git_dag_lab pack-ofs-verify
+PASS git-pack-ofs-delta-lab/v1 objects=2 pack_version=2 index_version=2 deltas=1 pack_sha1=736fdd73b58a6a68dfefec9288533a8d0a41a9e6 receipt_sha256=8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0
Index checksum acc33096abf3927200be57616ce87669d43efeeb · REF_DELTA false · thin pack false · arbitrary repository false · authentication claim false · network required false · exact byte identity scoped to the recorded Git build.
+
Generated offline · no JavaScript · no external assetsevidence/git-pack-ofs-delta-v1.json
+
+ + + diff --git a/docs/demo/git-pack-ofs-delta-v1/report.html b/docs/demo/git-pack-ofs-delta-v1/report.html new file mode 100644 index 0000000..3753d4b --- /dev/null +++ b/docs/demo/git-pack-ofs-delta-v1/report.html @@ -0,0 +1,36 @@ + + + + + + + Git OFS_DELTA evidence — verified offline report + + + +
+

Git pack lab · actual OFS_DELTA offline report

+

One backward edge.
Two verified blobs.

+

Real git pack-objects --delta-base-offset output is decoded without asking Git to replay it: the bounded delta program reconstructs the target, then index v2 independently binds both physical offsets and CRC32 rows.

+
PASS · report receipt 8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0
+
+
2logical blobs
+
1physical OFS_DELTA
+
77917actual pack bytes
+
12/12checks passed
+
+
+

Objects in physical pack order

formfixtureOIDoffsetstoredCRC32
fullline-1024-changed586c379527cbf256bb8dc7558e808ae5150a89591277824b6dfc85b
ofs-deltabaselined2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6778552684af9f53
Base → target
offset 12 → 77855; biased distance 77843 encoded as 83df13
586c379527cbf256bb8dc7558e808ae5150a8959
↓ independent copy/insert replay
d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6
+

Independent fail-closed gate

  • ✓all fixture objects present
  • ✓exactly one full entry
  • ✓exactly one ofs delta entry
  • ✓index checksum verified
  • ✓index crc32 matches pack
  • ✓index fanout matches sorted oids
  • ✓index offsets match pack
  • ✓ofs base entry bound
  • ✓ofs distance reencoded
  • ✓pack trailer verified
  • ✓reconstructed objects match fixture
  • ✓ref delta entries absent
+
+

Exact command provenance

$ git pack-objects --delta-base-offset --window=2 --depth=1 --threads=1 --compression=0 --no-reuse-delta --no-reuse-object --index-version=2 <private>/fixture
+stdin sha256=fa3a61ef14c4439502fdbfbaf0fe8a5b666b5ee54f70d48b99a14beb172cf952
+Git=git version 2.54.0
+frontend sha256=f54a87f6253aab09ed7b522bd78ddeab509105b1043076209d89127e55877a48

Real CLI receipt

$ python3 -B -m git_dag_lab pack-ofs-verify
+PASS git-pack-ofs-delta-lab/v1 objects=2 pack_version=2 index_version=2 deltas=1 pack_sha1=736fdd73b58a6a68dfefec9288533a8d0a41a9e6 receipt_sha256=8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0
Index checksum acc33096abf3927200be57616ce87669d43efeeb · REF_DELTA false · thin pack false · arbitrary repository false · authentication claim false · network required false · exact byte identity scoped to the recorded Git build.
+
Generated offline · no JavaScript · no external assetsevidence/git-pack-ofs-delta-v1.json
+
+ + diff --git a/docs/demo/git-pack-ofs-delta-v1/verify.txt b/docs/demo/git-pack-ofs-delta-v1/verify.txt new file mode 100644 index 0000000..a1ed888 --- /dev/null +++ b/docs/demo/git-pack-ofs-delta-v1/verify.txt @@ -0,0 +1 @@ +PASS git-pack-ofs-delta-lab/v1 objects=2 pack_version=2 index_version=2 deltas=1 pack_sha1=736fdd73b58a6a68dfefec9288533a8d0a41a9e6 receipt_sha256=8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0 diff --git a/evidence/git-pack-ofs-delta-v1.json b/evidence/git-pack-ofs-delta-v1.json new file mode 100644 index 0000000..1a52c35 --- /dev/null +++ b/evidence/git-pack-ofs-delta-v1.json @@ -0,0 +1 @@ +{"receipt":{"algorithm":"sha256","canonicalization":"UTF-8 JSON; sorted keys; compact separators","sha256":"8c2b8b07e9ab48492e9de13167c86eaf8835adcceecdcff958a6080a7ae540b0"},"report":{"checks":{"all_fixture_objects_present":true,"exactly_one_full_entry":true,"exactly_one_ofs_delta_entry":true,"index_checksum_verified":true,"index_crc32_matches_pack":true,"index_fanout_matches_sorted_oids":true,"index_offsets_match_pack":true,"ofs_base_entry_bound":true,"ofs_distance_reencoded":true,"pack_trailer_verified":true,"reconstructed_objects_match_fixture":true,"ref_delta_entries_absent":true},"command_trace":["init","hash-object","hash-object","pack-objects"],"fixture":{"changed_record":{"after":"1024:fedcba9876543210fedcba9876543210","before":"1024:0123456789abcdef0123456789abcdef","line_number":1024},"object_count":2,"objects":[{"label":"baseline","oid":"d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6","payload_sha256":"ea612ce3c3fb55f7a05ceecf1bcbf0ecf62fe298a8e387011e21f0d3d6353f1a","size":77824},{"label":"line-1024-changed","oid":"586c379527cbf256bb8dc7558e808ae5150a8959","payload_sha256":"e6bd970352e9ed0dc5389db5f9e4965c3d17e937881c69d4e7edf3df92e65dd4","size":77824}]},"index":{"bytes":1128,"index_sha1":"acc33096abf3927200be57616ce87669d43efeeb","nonzero_fanout_buckets":[{"cumulative":1,"prefix":"58","range_start":0},{"cumulative":2,"prefix":"d2","range_start":1}],"pack_sha1":"736fdd73b58a6a68dfefec9288533a8d0a41a9e6","sha256":"57fb9cfc0ac21a82384477f34b05e575b46f2a344bc10942dd4216603e5cb206","version":2},"object_format":"sha1","objects_in_pack_order":[{"crc32":"b6dfc85b","index_crc32":"b6dfc85b","index_offset":12,"label":"line-1024-changed","object_type":"blob","offset":12,"oid":"586c379527cbf256bb8dc7558e808ae5150a8959","packed_size":77843,"payload_sha256":"e6bd970352e9ed0dc5389db5f9e4965c3d17e937881c69d4e7edf3df92e65dd4","representation":"full","size":77824},{"base_label":"line-1024-changed","base_offset":12,"base_oid":"586c379527cbf256bb8dc7558e808ae5150a8959","crc32":"84af9f53","delta_depth":1,"index_crc32":"84af9f53","index_offset":77855,"label":"baseline","object_type":"blob","offset":77855,"ofs_distance":77843,"ofs_offset_bytes_hex":"83df13","oid":"d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6","packed_size":42,"payload_sha256":"ea612ce3c3fb55f7a05ceecf1bcbf0ecf62fe298a8e387011e21f0d3d6353f1a","representation":"ofs-delta","size":77824,"stored_size":26}],"pack":{"bytes":77917,"delta_count":1,"full_count":1,"max_delta_depth":1,"object_count":2,"ofs_delta_count":1,"ref_delta_count":0,"sha256":"080b1ca0b5cdb6c5e4cac746a48b8feb218aa7ec0b5583024a493d110c319a31","trailer_sha1":"736fdd73b58a6a68dfefec9288533a8d0a41a9e6","version":2},"pack_objects":{"normalized_argv":["git","pack-objects","--delta-base-offset","--window=2","--depth=1","--threads=1","--compression=0","--no-reuse-delta","--no-reuse-object","--index-version=2","/fixture"],"stdin_bytes":82,"stdin_oid_order":["586c379527cbf256bb8dc7558e808ae5150a8959","d2e8559ccc7b0bcfd193b7e9ad0b9d59982eb7c6"],"stdin_sha256":"fa3a61ef14c4439502fdbfbaf0fe8a5b666b5ee54f70d48b99a14beb172cf952"},"schema_version":"git-pack-ofs-delta-lab/v1","scope":{"arbitrary_repository_supported":false,"authentication_claim":false,"byte_identity_requires_same_git_build":true,"fixture_kind":"two deterministic similar synthetic blobs","git_pack_objects_executed":true,"network_required":false,"ofs_delta_supported":true,"ref_delta_supported":false,"thin_pack_supported":false}}} diff --git a/git_dag_lab/__init__.py b/git_dag_lab/__init__.py index 35ff0d1..8443ea4 100644 --- a/git_dag_lab/__init__.py +++ b/git_dag_lab/__init__.py @@ -1,7 +1,7 @@ """Deterministic experiments with real Git objects.""" from .lab import LabError, LabReport, git_object_oid, parse_commit, parse_tree, run_lab -from .pack import PackReport, parse_index, parse_pack, run_pack_lab +from .pack import PackReport, parse_index, parse_pack, run_ofs_pack_lab, run_pack_lab __all__ = [ "LabError", @@ -13,6 +13,7 @@ "parse_commit", "parse_tree", "run_lab", + "run_ofs_pack_lab", "run_pack_lab", ] diff --git a/git_dag_lab/cli.py b/git_dag_lab/cli.py index 95185c3..0e876c6 100644 --- a/git_dag_lab/cli.py +++ b/git_dag_lab/cli.py @@ -9,7 +9,7 @@ from typing import TextIO from .lab import LabError, run_lab -from .pack import run_pack_lab +from .pack import run_ofs_pack_lab, run_pack_lab def build_parser() -> argparse.ArgumentParser: @@ -42,6 +42,20 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="emit canonical JSON on one line instead of indented JSON", ) + + subparsers.add_parser( + "pack-ofs-verify", + help="build and independently verify a real OFS-delta pack/index pair", + ) + ofs_inspect_parser = subparsers.add_parser( + "pack-ofs-inspect", + help="print the complete machine-readable OFS-delta evidence document", + ) + ofs_inspect_parser.add_argument( + "--compact", + action="store_true", + help="emit canonical JSON on one line instead of indented JSON", + ) return parser @@ -59,7 +73,9 @@ def main( args = build_parser().parse_args(argv) try: - if args.command.startswith("pack-"): + if args.command.startswith("pack-ofs-"): + report = run_ofs_pack_lab(root if root is not None else Path.cwd()) + elif args.command.startswith("pack-"): report = run_pack_lab(root if root is not None else Path.cwd()) else: report = run_lab(root if root is not None else Path.cwd()) @@ -67,7 +83,7 @@ def main( errors.write(f"ERROR git-dag-lab: {exc}\n") return 1 - if args.command in {"verify", "pack-verify"}: + if args.command in {"verify", "pack-verify", "pack-ofs-verify"}: output.write(report.receipt_line + "\n") else: output.write(report.to_json(pretty=not args.compact) + "\n") diff --git a/git_dag_lab/pack.py b/git_dag_lab/pack.py index 606b50b..d943c7d 100644 --- a/git_dag_lab/pack.py +++ b/git_dag_lab/pack.py @@ -1,8 +1,8 @@ """Build and independently verify a bounded, deterministic Git pack/index pair. -The fixture uses real ``git pack-objects`` output but validates the pack v2 and -index v2 bytes with Python's standard library. Delta entries are disabled for -this first closed subset and rejected by the parser. +The baseline fixture uses real ``git pack-objects`` output and validates pack +v2/index v2 bytes with Python's standard library. The parser additionally +supports a bounded OFS_DELTA subset; REF_DELTA remains explicitly rejected. """ from __future__ import annotations @@ -34,6 +34,7 @@ ) PACK_SCHEMA_VERSION = "git-pack-index-lab/v1" +OFS_PACK_SCHEMA_VERSION = "git-pack-ofs-delta-lab/v1" PACK_VERSION = 2 INDEX_VERSION = 2 PACK_MAGIC = b"PACK" @@ -41,6 +42,11 @@ MAX_PACK_BYTES = 1_048_576 MAX_PACK_OBJECTS = 64 MAX_OBJECT_BYTES = 262_144 +MAX_DELTA_DEPTH = 4 +MAX_DELTA_INSTRUCTIONS = 4_096 +MAX_DELTA_SIZE_BYTES = 3 +MAX_OFS_OFFSET_BYTES = 3 +MAX_TOTAL_EXPANDED_BYTES = 16_777_216 PACK_BLOBS = ( ("binary-header", bytes(range(32))), @@ -50,12 +56,29 @@ b"fanout tables map object-id prefixes to sorted index ranges\n", ), ) -_TYPE_BY_CODE = {1: "commit", 2: "tree", 3: "blob", 4: "tag"} +_OFS_RECORD_BEFORE = b"1024:0123456789abcdef0123456789abcdef\n" +_OFS_RECORD_AFTER = b"1024:fedcba9876543210fedcba9876543210\n" +_OFS_BASE_PAYLOAD = b"".join( + f"{line:04d}:0123456789abcdef0123456789abcdef\n".encode("ascii") + for line in range(2_048) +) +_OFS_TARGET_PAYLOAD = _OFS_BASE_PAYLOAD.replace( + _OFS_RECORD_BEFORE, + _OFS_RECORD_AFTER, + 1, +) +OFS_PACK_BLOBS = ( + ("baseline", _OFS_BASE_PAYLOAD), + ("line-1024-changed", _OFS_TARGET_PAYLOAD), +) + +_TYPE_BY_CODE = {1: "commit", 2: "tree", 3: "blob"} +_ENTRY_KIND_BY_CODE = {**_TYPE_BY_CODE, 6: "ofs-delta"} @dataclass(frozen=True, slots=True) class PackEntry: - """One independently decoded non-delta pack entry.""" + """One independently decoded full or OFS-delta pack entry.""" crc32: int object_type: str @@ -64,9 +87,14 @@ class PackEntry: packed_size: int payload_sha256: str size: int + base_offset: int | None = None + base_oid: str | None = None + delta_depth: int = 0 + representation: str = "full" + stored_size: int = 0 def as_dict(self) -> dict[str, object]: - return { + payload: dict[str, object] = { "crc32": f"{self.crc32:08x}", "object_type": self.object_type, "offset": self.offset, @@ -75,6 +103,17 @@ def as_dict(self) -> dict[str, object]: "payload_sha256": self.payload_sha256, "size": self.size, } + if self.representation != "full": + payload.update( + { + "base_offset": self.base_offset, + "base_oid": self.base_oid, + "delta_depth": self.delta_depth, + "representation": self.representation, + "stored_size": self.stored_size, + } + ) + return payload @dataclass(frozen=True, slots=True) @@ -86,6 +125,16 @@ class ParsedPack: version: int +@dataclass(frozen=True, slots=True) +class _ResolvedObject: + """One bounded canonical object available as a delta base.""" + + depth: int + object_type: str + oid: str + payload: bytes + + @dataclass(frozen=True, slots=True) class IndexEntry: """One verified index v2 row.""" @@ -136,11 +185,11 @@ def receipt_line(self) -> str: pack = self.payload["pack"] index = self.payload["index"] return ( - f"PASS {PACK_SCHEMA_VERSION} " + f"PASS {self.payload['schema_version']} " f"objects={pack['object_count']} " f"pack_version={pack['version']} " f"index_version={index['version']} " - "deltas=0 " + f"deltas={pack['delta_count']} " f"pack_sha1={pack['trailer_sha1']} " f"receipt_sha256={self.receipt_sha256}" ) @@ -228,9 +277,9 @@ def _decode_entry_header( first = content[offset] offset += 1 type_code = (first >> 4) & 0x07 - if type_code not in _TYPE_BY_CODE: - if type_code in {6, 7}: - raise VerificationError("delta pack entries are outside the reviewed subset") + if type_code == 7: + raise VerificationError("REF_DELTA pack entries are outside the reviewed subset") + if type_code not in _ENTRY_KIND_BY_CODE: raise VerificationError("pack entry type is invalid") size = first & 0x0F @@ -244,8 +293,182 @@ def _decode_entry_header( size |= (current & 0x7F) << shift shift += 7 if size > MAX_OBJECT_BYTES: - raise VerificationError("pack entry expands beyond the reviewed bound") - return _TYPE_BY_CODE[type_code], size, offset + raise VerificationError("pack entry data exceeds the reviewed bound") + return _ENTRY_KIND_BY_CODE[type_code], size, offset + + +def _inflate_entry( + content: bytes, + offset: int, + *, + declared_size: int, + end: int, +) -> tuple[bytes, int]: + source = content[offset:end] + inflater = zlib.decompressobj() + try: + payload = inflater.decompress(source, declared_size + 1) + except zlib.error as exc: + raise VerificationError("pack entry zlib stream is invalid") from exc + if len(payload) > declared_size: + raise VerificationError("pack entry expands beyond its declared size") + if not inflater.eof or inflater.unconsumed_tail: + raise VerificationError("pack entry zlib stream is incomplete or oversized") + consumed = len(source) - len(inflater.unused_data) + if consumed < 1: + raise VerificationError("pack entry has an empty zlib stream") + next_offset = offset + consumed + if next_offset > end or len(payload) != declared_size: + raise VerificationError("pack entry size does not match its payload") + return payload, next_offset + + +def _encode_ofs_distance(distance: int) -> bytes: + if distance < 1: + raise ValueError("OFS_DELTA distance must be positive") + encoded = bytearray((distance & 0x7F,)) + remaining = distance + while remaining >> 7: + remaining = (remaining >> 7) - 1 + encoded.append(0x80 | (remaining & 0x7F)) + encoded.reverse() + return bytes(encoded) + + +def _decode_ofs_base_offset( + content: bytes, + offset: int, + *, + end: int, + entry_offset: int, +) -> tuple[int, int]: + start = offset + if offset >= end: + raise VerificationError("OFS_DELTA base offset is truncated") + current = content[offset] + offset += 1 + distance = current & 0x7F + while current & 0x80: + if offset >= end or offset - start >= MAX_OFS_OFFSET_BYTES: + raise VerificationError("OFS_DELTA base offset is malformed") + current = content[offset] + offset += 1 + distance = ((distance + 1) << 7) | (current & 0x7F) + if distance > MAX_PACK_BYTES: + raise VerificationError("OFS_DELTA base offset exceeds the pack bound") + encoded = content[start:offset] + if distance < 1 or encoded != _encode_ofs_distance(distance): + raise VerificationError("OFS_DELTA base offset is non-canonical") + base_offset = entry_offset - distance + if base_offset < 12 or base_offset >= entry_offset: + raise VerificationError("OFS_DELTA base offset is out of bounds") + return base_offset, offset + + +def _encode_delta_size(value: int) -> bytes: + if value < 0: + raise ValueError("delta size must be non-negative") + encoded = bytearray() + remaining = value + while True: + current = remaining & 0x7F + remaining >>= 7 + if remaining: + current |= 0x80 + encoded.append(current) + if not remaining: + return bytes(encoded) + + +def _decode_delta_size( + program: bytes, + offset: int, + *, + label: str, +) -> tuple[int, int]: + start = offset + value = 0 + shift = 0 + while True: + if offset >= len(program) or offset - start >= MAX_DELTA_SIZE_BYTES: + raise VerificationError(f"{label} is truncated or oversized") + current = program[offset] + offset += 1 + value |= (current & 0x7F) << shift + if value > MAX_OBJECT_BYTES: + raise VerificationError(f"{label} exceeds the reviewed bound") + if not current & 0x80: + break + shift += 7 + if program[start:offset] != _encode_delta_size(value): + raise VerificationError(f"{label} is non-canonical") + return value, offset + + +def _apply_delta(base: bytes, program: bytes) -> bytes: + if type(base) is not bytes or type(program) is not bytes: + raise VerificationError("delta inputs must be exact bytes") + if len(program) < 4: + raise VerificationError("delta program is shorter than Git's minimum") + base_size, offset = _decode_delta_size( + program, + 0, + label="delta base size", + ) + result_size, offset = _decode_delta_size( + program, + offset, + label="delta result size", + ) + if base_size != len(base): + raise VerificationError("delta base size does not match its object") + if result_size > MAX_OBJECT_BYTES: + raise VerificationError("delta result exceeds the reviewed bound") + + output = bytearray() + instructions = 0 + while offset < len(program): + instructions += 1 + if instructions > MAX_DELTA_INSTRUCTIONS: + raise VerificationError("delta instruction count exceeds the reviewed bound") + opcode = program[offset] + offset += 1 + if opcode & 0x80: + copy_offset = 0 + copy_size = 0 + for bit, shift in ((0x01, 0), (0x02, 8), (0x04, 16), (0x08, 24)): + if opcode & bit: + if offset >= len(program): + raise VerificationError("delta copy offset is truncated") + copy_offset |= program[offset] << shift + offset += 1 + for bit, shift in ((0x10, 0), (0x20, 8), (0x40, 16)): + if opcode & bit: + if offset >= len(program): + raise VerificationError("delta copy size is truncated") + copy_size |= program[offset] << shift + offset += 1 + if copy_size == 0: + copy_size = 0x10000 + if copy_offset > len(base) or copy_size > len(base) - copy_offset: + raise VerificationError("delta copy range exceeds its base object") + if copy_size > result_size - len(output): + raise VerificationError("delta copy exceeds its declared result") + output.extend(base[copy_offset : copy_offset + copy_size]) + elif opcode: + literal_size = opcode & 0x7F + literal_end = offset + literal_size + if literal_end > len(program): + raise VerificationError("delta literal is truncated") + if literal_size > result_size - len(output): + raise VerificationError("delta literal exceeds its declared result") + output.extend(program[offset:literal_end]) + offset = literal_end + else: + raise VerificationError("delta opcode zero is reserved") + if len(output) != result_size: + raise VerificationError("delta result size does not match its instructions") + return bytes(output) def parse_pack(content: bytes) -> ParsedPack: @@ -274,37 +497,65 @@ def parse_pack(content: bytes) -> ParsedPack: entries: list[PackEntry] = [] offset = 12 seen: set[str] = set() + resolved_by_offset: dict[int, _ResolvedObject] = {} + expanded_bytes = 0 for _ in range(count): entry_start = offset - object_type, declared_size, payload_start = _decode_entry_header( + entry_kind, declared_size, payload_start = _decode_entry_header( content, offset, end=pack_end, ) - inflater = zlib.decompressobj() - try: - payload = inflater.decompress( - content[payload_start:pack_end], - MAX_OBJECT_BYTES + 1, + base_offset: int | None = None + base_oid: str | None = None + depth = 0 + representation = "full" + if entry_kind == "ofs-delta": + base_offset, payload_start = _decode_ofs_base_offset( + content, + payload_start, + end=pack_end, + entry_offset=entry_start, ) - except zlib.error as exc: - raise VerificationError("pack entry zlib stream is invalid") from exc - if len(payload) > MAX_OBJECT_BYTES: - raise VerificationError("pack entry expands beyond the reviewed bound") - if not inflater.eof or inflater.unconsumed_tail: - raise VerificationError("pack entry zlib stream is incomplete or oversized") - consumed = pack_end - payload_start - len(inflater.unused_data) - if consumed < 1: - raise VerificationError("pack entry has an empty zlib stream") - offset = payload_start + consumed - if offset > pack_end or len(payload) != declared_size: - raise VerificationError("pack entry size does not match its payload") + stored_payload, offset = _inflate_entry( + content, + payload_start, + declared_size=declared_size, + end=pack_end, + ) + if entry_kind == "ofs-delta": + assert base_offset is not None + base = resolved_by_offset.get(base_offset) + if base is None: + raise VerificationError( + "OFS_DELTA base is not an earlier pack-entry boundary" + ) + depth = base.depth + 1 + if depth > MAX_DELTA_DEPTH: + raise VerificationError("OFS_DELTA chain exceeds the reviewed depth") + payload = _apply_delta(base.payload, stored_payload) + object_type = base.object_type + base_oid = base.oid + representation = entry_kind + else: + payload = stored_payload + object_type = entry_kind + + expanded_bytes += len(payload) + if expanded_bytes > MAX_TOTAL_EXPANDED_BYTES: + raise VerificationError("pack expands beyond the aggregate reviewed bound") oid = git_object_oid(object_type, payload) if oid in seen: raise VerificationError("pack contains a duplicate logical object") seen.add(oid) packed = content[entry_start:offset] + resolved_by_offset[entry_start] = _ResolvedObject( + depth=depth, + object_type=object_type, + oid=oid, + payload=payload, + ) entries.append( PackEntry( crc32=binascii.crc32(packed) & 0xFFFFFFFF, @@ -314,6 +565,11 @@ def parse_pack(content: bytes) -> ParsedPack: packed_size=len(packed), payload_sha256=hashlib.sha256(payload).hexdigest(), size=len(payload), + base_offset=base_offset, + base_oid=base_oid, + delta_depth=depth, + representation=representation, + stored_size=declared_size, ) ) if offset != pack_end: @@ -576,6 +832,251 @@ def _build_and_verify_pack(runner: _GitRunner, private: Path) -> PackReport: return PackReport(payload=_deep_freeze(payload), receipt_sha256=receipt) +def _build_and_verify_ofs_pack(runner: _GitRunner, private: Path) -> PackReport: + runner.initialize() + if ( + len(_OFS_BASE_PAYLOAD) != 77_824 + or len(_OFS_TARGET_PAYLOAD) != 77_824 + or _OFS_BASE_PAYLOAD.count(_OFS_RECORD_BEFORE) != 1 + or _OFS_TARGET_PAYLOAD.count(_OFS_RECORD_AFTER) != 1 + ): + raise VerificationError("fixed OFS fixture bytes are not exact") + + expected_payloads: dict[str, bytes] = {} + labels_by_oid: dict[str, str] = {} + for label, payload in OFS_PACK_BLOBS: + oid = _store_blob(runner, payload) + expected_payloads[oid] = payload + labels_by_oid[oid] = label + if len(expected_payloads) != len(OFS_PACK_BLOBS): + raise VerificationError("fixed OFS fixture contains duplicate objects") + + ordered_oids = tuple(sorted(expected_payloads)) + object_input = b"".join(f"{oid}\n".encode("ascii") for oid in ordered_oids) + prefix = private / "fixture" + pack_arguments = ( + "--delta-base-offset", + "--window=2", + "--depth=1", + "--threads=1", + "--compression=0", + "--no-reuse-delta", + "--no-reuse-object", + "--index-version=2", + os.fspath(prefix), + ) + result = runner.run( + "pack-objects", + *pack_arguments, + stdin=object_input, + ) + if result.stderr: + raise VerificationError("git pack-objects produced unexpected diagnostics") + try: + pack_name = result.stdout.decode("ascii").strip() + except UnicodeDecodeError as exc: + raise VerificationError("git pack-objects returned a non-ASCII checksum") from exc + if ( + len(pack_name) != 40 + or any(character not in "0123456789abcdef" for character in pack_name) + ): + raise VerificationError("git pack-objects returned an invalid checksum") + + pack_path = Path(f"{prefix}-{pack_name}.pack") + index_path = Path(f"{prefix}-{pack_name}.idx") + pack_bytes = _read_regular_file(pack_path, label="generated OFS pack") + index_bytes = _read_regular_file(index_path, label="generated OFS index") + parsed_pack = parse_pack(pack_bytes) + parsed_index = parse_index(index_bytes) + _cross_check(parsed_pack, parsed_index) + if parsed_pack.trailer_sha1 != pack_name: + raise VerificationError("git OFS pack name does not match the verified trailer") + if {entry.oid for entry in parsed_pack.entries} != set(expected_payloads): + raise VerificationError("generated OFS pack object inventory is not exact") + + full_entries = tuple( + entry for entry in parsed_pack.entries if entry.representation == "full" + ) + ofs_entries = tuple( + entry for entry in parsed_pack.entries if entry.representation == "ofs-delta" + ) + if len(full_entries) != 1 or len(ofs_entries) != 1: + raise VerificationError("Git did not produce exactly one full and one OFS entry") + base_entry = full_entries[0] + delta_entry = ofs_entries[0] + if ( + delta_entry.base_offset != base_entry.offset + or delta_entry.base_oid != base_entry.oid + or delta_entry.delta_depth != 1 + ): + raise VerificationError("generated OFS entry does not bind the exact full base") + + for entry in parsed_pack.entries: + payload = expected_payloads[entry.oid] + if ( + entry.object_type != "blob" + or entry.size != len(payload) + or entry.payload_sha256 != hashlib.sha256(payload).hexdigest() + ): + raise VerificationError("reconstructed OFS object differs from the fixture") + + pack_end = len(pack_bytes) - 20 + entry_kind, _, ofs_start = _decode_entry_header( + pack_bytes, + delta_entry.offset, + end=pack_end, + ) + decoded_base, ofs_end = _decode_ofs_base_offset( + pack_bytes, + ofs_start, + end=pack_end, + entry_offset=delta_entry.offset, + ) + ofs_distance = delta_entry.offset - base_entry.offset + expected_ofs_bytes = _encode_ofs_distance(ofs_distance) + observed_ofs_bytes = pack_bytes[ofs_start:ofs_end] + if ( + entry_kind != "ofs-delta" + or decoded_base != base_entry.offset + or observed_ofs_bytes != expected_ofs_bytes + ): + raise VerificationError("generated OFS distance does not re-encode exactly") + + index_rows = {entry.oid: entry for entry in parsed_index.entries} + objects = [] + for entry in parsed_pack.entries: + indexed = index_rows[entry.oid] + row: dict[str, object] = { + **entry.as_dict(), + "index_crc32": f"{indexed.crc32:08x}", + "index_offset": indexed.offset, + "label": labels_by_oid[entry.oid], + "representation": entry.representation, + } + if entry.representation == "ofs-delta": + row.update( + { + "base_label": labels_by_oid[base_entry.oid], + "ofs_distance": ofs_distance, + "ofs_offset_bytes_hex": observed_ofs_bytes.hex(), + } + ) + objects.append(row) + + nonzero_buckets = [ + { + "cumulative": parsed_index.fanout[bucket], + "prefix": f"{bucket:02x}", + "range_start": 0 if bucket == 0 else parsed_index.fanout[bucket - 1], + } + for bucket in range(256) + if ( + parsed_index.fanout[bucket] + != (0 if bucket == 0 else parsed_index.fanout[bucket - 1]) + ) + ] + payload: dict[str, object] = { + "checks": { + "all_fixture_objects_present": True, + "exactly_one_full_entry": True, + "exactly_one_ofs_delta_entry": True, + "index_checksum_verified": True, + "index_crc32_matches_pack": True, + "index_fanout_matches_sorted_oids": True, + "index_offsets_match_pack": True, + "ofs_base_entry_bound": True, + "ofs_distance_reencoded": True, + "pack_trailer_verified": True, + "reconstructed_objects_match_fixture": True, + "ref_delta_entries_absent": True, + }, + "command_trace": list(runner.trace), + "fixture": { + "changed_record": { + "after": _OFS_RECORD_AFTER.decode("ascii").rstrip("\n"), + "before": _OFS_RECORD_BEFORE.decode("ascii").rstrip("\n"), + "line_number": 1024, + }, + "object_count": len(OFS_PACK_BLOBS), + "objects": [ + { + "label": label, + "oid": git_object_oid("blob", body), + "payload_sha256": hashlib.sha256(body).hexdigest(), + "size": len(body), + } + for label, body in OFS_PACK_BLOBS + ], + }, + "index": { + "bytes": len(index_bytes), + "index_sha1": parsed_index.index_sha1, + "nonzero_fanout_buckets": nonzero_buckets, + "pack_sha1": parsed_index.pack_sha1, + "sha256": hashlib.sha256(index_bytes).hexdigest(), + "version": parsed_index.version, + }, + "object_format": "sha1", + "objects_in_pack_order": objects, + "pack": { + "bytes": len(pack_bytes), + "delta_count": len(ofs_entries), + "full_count": len(full_entries), + "max_delta_depth": max(entry.delta_depth for entry in parsed_pack.entries), + "object_count": len(parsed_pack.entries), + "ofs_delta_count": len(ofs_entries), + "ref_delta_count": 0, + "sha256": hashlib.sha256(pack_bytes).hexdigest(), + "trailer_sha1": parsed_pack.trailer_sha1, + "version": parsed_pack.version, + }, + "pack_objects": { + "normalized_argv": [ + "git", + "pack-objects", + *pack_arguments[:-1], + "/fixture", + ], + "stdin_bytes": len(object_input), + "stdin_oid_order": list(ordered_oids), + "stdin_sha256": hashlib.sha256(object_input).hexdigest(), + }, + "scope": { + "arbitrary_repository_supported": False, + "authentication_claim": False, + "byte_identity_requires_same_git_build": True, + "fixture_kind": "two deterministic similar synthetic blobs", + "git_pack_objects_executed": True, + "network_required": False, + "ofs_delta_supported": True, + "ref_delta_supported": False, + "thin_pack_supported": False, + }, + "schema_version": OFS_PACK_SCHEMA_VERSION, + } + receipt = hashlib.sha256(_canonical_json(payload)).hexdigest() + return PackReport(payload=_deep_freeze(payload), receipt_sha256=receipt) + + +def run_ofs_pack_lab(root: Path | str = ".") -> PackReport: + """Build and independently verify a real OFS-delta pack/index pair.""" + + try: + validated_root = _validate_root(Path(root)) + git = _find_git() + with tempfile.TemporaryDirectory( + prefix=".git-pack-ofs-delta-lab-", + dir=validated_root, + ) as private_name: + workspace = _new_workspace(validated_root, Path(private_name)) + runner = _GitRunner(git, workspace) + return _build_and_verify_ofs_pack(runner, workspace.private) + except LabError: + raise + except (OSError, TypeError, ValueError) as exc: + raise LabError("isolated OFS pack lab setup failed") from exc + + def run_pack_lab(root: Path | str = ".") -> PackReport: """Build and verify a real pack/index pair below ``root``, then remove it.""" @@ -597,9 +1098,13 @@ def run_pack_lab(root: Path | str = ".") -> PackReport: __all__ = [ "INDEX_VERSION", + "MAX_DELTA_DEPTH", + "MAX_DELTA_INSTRUCTIONS", "MAX_OBJECT_BYTES", + "MAX_TOTAL_EXPANDED_BYTES", "MAX_PACK_BYTES", "MAX_PACK_OBJECTS", + "OFS_PACK_SCHEMA_VERSION", "PACK_SCHEMA_VERSION", "PACK_VERSION", "IndexEntry", @@ -609,5 +1114,6 @@ def run_pack_lab(root: Path | str = ".") -> PackReport: "ParsedPack", "parse_index", "parse_pack", + "run_ofs_pack_lab", "run_pack_lab", ] diff --git a/tests/test_ofs_evidence.py b/tests/test_ofs_evidence.py new file mode 100644 index 0000000..234fa65 --- /dev/null +++ b/tests/test_ofs_evidence.py @@ -0,0 +1,272 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import re +import unittest + +from tools import generate_ofs_evidence as evidence + + +ROOT = Path(__file__).resolve().parents[1] + + +class OfsEvidenceTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.generated = evidence.build_artifacts(allow_missing_screenshot=False) + cls.document = json.loads(cls.generated[evidence.EVIDENCE_PATH]) + cls.manifest = json.loads(cls.generated[evidence.MANIFEST_PATH]) + cls.receipt = cls.document["receipt"]["sha256"] + + def test_checked_in_generated_files_are_current(self) -> None: + for path, expected in self.generated.items(): + with self.subTest(path=path.as_posix()): + self.assertEqual((ROOT / path).read_bytes(), expected) + + def test_two_fresh_evidence_builds_are_byte_identical(self) -> None: + first = evidence.build_artifacts(allow_missing_screenshot=False) + second = evidence.build_artifacts(allow_missing_screenshot=False) + self.assertEqual(first, second) + + def test_expected_generated_inventory_is_exact(self) -> None: + self.assertEqual( + set(self.generated), + { + evidence.EVIDENCE_PATH, + evidence.VERIFY_PATH, + evidence.INSPECT_PATH, + evidence.REPORT_PATH, + evidence.MANIFEST_PATH, + evidence.RECONSTRUCTION_PATH, + evidence.WORKFLOW_PATH, + evidence.CLI_PATH, + }, + ) + rows = {row["path"] for row in self.manifest["artifacts"]} + self.assertEqual( + rows, + { + evidence.EVIDENCE_PATH.as_posix(), + evidence.VERIFY_PATH.as_posix(), + evidence.INSPECT_PATH.as_posix(), + evidence.REPORT_PATH.as_posix(), + evidence.RECONSTRUCTION_PATH.as_posix(), + evidence.WORKFLOW_PATH.as_posix(), + evidence.CLI_PATH.as_posix(), + evidence.SCREENSHOT_PATH.as_posix(), + evidence.RENDERED_DOM_PATH.as_posix(), + evidence.ATTESTATION_PATH.as_posix(), + }, + ) + + def test_compact_pretty_and_verify_outputs_share_one_receipt(self) -> None: + compact = json.loads((ROOT / evidence.EVIDENCE_PATH).read_text()) + pretty = json.loads((ROOT / evidence.INSPECT_PATH).read_text()) + verify = (ROOT / evidence.VERIFY_PATH).read_text() + self.assertEqual(compact, pretty) + self.assertEqual(compact, self.document) + self.assertIn(f"receipt_sha256={self.receipt}\n", verify) + self.assertIn("objects=2", verify) + self.assertIn("deltas=1", verify) + report = self.document["report"] + self.assertEqual(report["pack"]["ofs_delta_count"], 1) + self.assertEqual(report["pack"]["ref_delta_count"], 0) + self.assertTrue(all(report["checks"].values())) + + def test_manifest_binds_artifacts_sources_and_git_build(self) -> None: + rows = {row["path"]: row for row in self.manifest["artifacts"]} + for relative, row in rows.items(): + content = (ROOT / relative).read_bytes() + self.assertEqual(row["size"], len(content)) + self.assertEqual(row["sha256"], hashlib.sha256(content).hexdigest()) + sources = {row["path"]: row for row in self.manifest["sources"]} + self.assertEqual( + set(sources), + { + "git_dag_lab/__init__.py", + "git_dag_lab/__main__.py", + "git_dag_lab/cli.py", + "git_dag_lab/lab.py", + "git_dag_lab/pack.py", + "tools/generate_evidence.py", + "tools/generate_pack_evidence.py", + "tools/generate_ofs_evidence.py", + "tools/capture_ofs_report.sh", + }, + ) + for relative, row in sources.items(): + content = (ROOT / relative).read_bytes() + self.assertEqual(row["size"], len(content)) + self.assertEqual(row["sha256"], hashlib.sha256(content).hexdigest()) + git_build = self.manifest["git_build"] + self.assertRegex(git_build["version"], r"^git version [0-9]+(?:\.[0-9]+){1,3}$") + self.assertRegex(git_build["frontend_sha256"], r"^[0-9a-f]{64}$") + self.assertRegex(git_build["pack_objects_sha256"], r"^[0-9a-f]{64}$") + self.assertEqual( + git_build["byte_identity_scope"], + "same recorded Git build", + ) + + def test_browser_capture_and_attestation_are_current(self) -> None: + screenshot = (ROOT / evidence.SCREENSHOT_PATH).read_bytes() + rendered_dom = (ROOT / evidence.RENDERED_DOM_PATH).read_bytes() + attestation_bytes = (ROOT / evidence.ATTESTATION_PATH).read_bytes() + self.assertEqual( + evidence._parse_pack_png(screenshot), + evidence.PNG_DIMENSIONS, + ) + capture = self.manifest["capture"] + self.assertEqual(capture["status"], "attested") + self.assertEqual(capture["screenshot_sha256"], hashlib.sha256(screenshot).hexdigest()) + self.assertEqual( + capture["rendered_dom_sha256"], + hashlib.sha256(rendered_dom).hexdigest(), + ) + attestation = json.loads(attestation_bytes) + payload = attestation["attestation"] + canonical = json.dumps( + payload, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + self.assertEqual( + attestation["receipt"]["sha256"], + hashlib.sha256(canonical).hexdigest(), + ) + self.assertEqual( + capture["attestation_receipt_sha256"], + attestation["receipt"]["sha256"], + ) + self.assertEqual(payload["isolation"]["network"], "none") + self.assertEqual(payload["isolation"]["root_filesystem"], "read-only") + self.assertTrue(payload["isolation"]["no_new_privileges"]) + + def test_svg_visuals_are_accessible_receipt_bound_and_factual(self) -> None: + report = self.document["report"] + full, delta = report["objects_in_pack_order"] + for path in ( + evidence.RECONSTRUCTION_PATH, + evidence.WORKFLOW_PATH, + evidence.CLI_PATH, + ): + text = (ROOT / path).read_text() + with self.subTest(path=path.as_posix()): + self.assertIn(" None: + report = (ROOT / evidence.REPORT_PATH).read_text() + self.assertNotIn(" None: + verify = (ROOT / evidence.VERIFY_PATH).read_bytes() + mutations = [] + + changed_count = deepcopy(self.document) + changed_count["report"]["pack"]["delta_count"] = 0 + mutations.append(changed_count) + + changed_distance = deepcopy(self.document) + changed_distance["report"]["objects_in_pack_order"][1]["ofs_distance"] += 1 + mutations.append(changed_distance) + + changed_command = deepcopy(self.document) + changed_command["report"]["pack_objects"]["normalized_argv"][2] = "--window=99" + mutations.append(changed_command) + + changed_scope = deepcopy(self.document) + changed_scope["report"]["scope"]["authentication_claim"] = True + mutations.append(changed_scope) + + for mutation in mutations: + with self.assertRaises(evidence.EvidenceError): + evidence._validate_document(mutation, verify) + + def test_public_evidence_has_no_host_secret_or_personal_markers(self) -> None: + text_paths = [ + evidence.EVIDENCE_PATH, + evidence.VERIFY_PATH, + evidence.INSPECT_PATH, + evidence.REPORT_PATH, + evidence.MANIFEST_PATH, + evidence.RENDERED_DOM_PATH, + evidence.ATTESTATION_PATH, + evidence.RECONSTRUCTION_PATH, + evidence.WORKFLOW_PATH, + evidence.CLI_PATH, + ] + combined = "\n".join((ROOT / path).read_text() for path in text_paths) + for marker in ( + "/home/", + "@gmail.com", + "github.com/", + "AKIA", + "ghp_", + "github_pat_", + "Omar Ibrahim", + ): + self.assertNotIn(marker.lower(), combined.lower()) + + def test_readme_exposes_the_real_ofs_workflow_and_visuals(self) -> None: + readme = (ROOT / "README.md").read_text() + for path in ( + evidence.EVIDENCE_PATH, + evidence.SCREENSHOT_PATH, + evidence.RECONSTRUCTION_PATH, + evidence.WORKFLOW_PATH, + evidence.CLI_PATH, + evidence.MANIFEST_PATH, + ): + self.assertIn(path.as_posix(), readme) + self.assertIn("python3 -m git_dag_lab pack-ofs-verify", readme) + self.assertIn("python3 -m git_dag_lab pack-ofs-inspect", readme) + self.assertIn("python3 -B tools/generate_ofs_evidence.py --check", readme) + self.assertIn("tools/capture_ofs_report.sh", readme) + self.assertIn("12/12 OFS checks", readme) + + def test_capture_toolchain_is_digest_pinned_and_network_disabled(self) -> None: + script = (ROOT / "tools/capture_ofs_report.sh").read_text() + self.assertRegex( + script, + r"mcr\.microsoft\.com/playwright@sha256:[0-9a-f]{64}", + ) + self.assertIn("--network none", script) + self.assertIn("--pull=never", script) + self.assertIn("--read-only", script) + self.assertIn("--cap-drop ALL", script) + self.assertIn("--security-opt no-new-privileges", script) + self.assertIn( + 'chmod 0600 "$temporary_root/output/git-pack-ofs-report.png"', + script, + ) + self.assertNotRegex(script, re.compile(r"(?m)^\s*curl\b|\bwget\b")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pack_lab.py b/tests/test_pack_lab.py index 8a4c843..b380b68 100644 --- a/tests/test_pack_lab.py +++ b/tests/test_pack_lab.py @@ -17,12 +17,18 @@ from git_dag_lab.pack import ( INDEX_MAGIC, INDEX_VERSION, + MAX_DELTA_DEPTH, + MAX_DELTA_INSTRUCTIONS, + MAX_OBJECT_BYTES, MAX_PACK_BYTES, + MAX_TOTAL_EXPANDED_BYTES, + OFS_PACK_SCHEMA_VERSION, PACK_MAGIC, PACK_SCHEMA_VERSION, PACK_VERSION, parse_index, parse_pack, + run_ofs_pack_lab, run_pack_lab, ) @@ -69,6 +75,68 @@ def _resign_index(prefix: bytes) -> bytes: return prefix + hashlib.sha1(prefix, usedforsecurity=False).digest() +def _delta_size(value: int) -> bytes: + output = bytearray() + while True: + current = value & 0x7F + value >>= 7 + if value: + current |= 0x80 + output.append(current) + if not value: + return bytes(output) + + +def _ofs_distance(distance: int) -> bytes: + output = bytearray((distance & 0x7F,)) + while distance >> 7: + distance = (distance >> 7) - 1 + output.append(0x80 | (distance & 0x7F)) + output.reverse() + return bytes(output) + + +def _copy_instruction(*, offset: int = 0, size: int) -> bytes: + command = 0x80 + parameters = bytearray() + for bit, shift in ((0x01, 0), (0x02, 8), (0x04, 16), (0x08, 24)): + value = (offset >> shift) & 0xFF + if value: + command |= bit + parameters.append(value) + if size != 0x10000: + for bit, shift in ((0x10, 0), (0x20, 8), (0x40, 16)): + value = (size >> shift) & 0xFF + if value: + command |= bit + parameters.append(value) + return bytes((command,)) + bytes(parameters) + + +def _program(base_size: int, result_size: int, instructions: bytes) -> bytes: + return _delta_size(base_size) + _delta_size(result_size) + instructions + + +def _build_ofs_pack( + base_payload: bytes, + deltas: tuple[tuple[int, bytes], ...], +) -> tuple[bytes, tuple[int, ...]]: + body = bytearray(PACK_MAGIC) + body.extend(PACK_VERSION.to_bytes(4, "big")) + body.extend((1 + len(deltas)).to_bytes(4, "big")) + offsets = [len(body)] + body.extend(_entry_header(3, len(base_payload))) + body.extend(zlib.compress(base_payload, level=9)) + for base_index, program in deltas: + entry_offset = len(body) + distance = entry_offset - offsets[base_index] + body.extend(_entry_header(6, len(program))) + body.extend(_ofs_distance(distance)) + body.extend(zlib.compress(program, level=9)) + offsets.append(entry_offset) + return _resign_pack(bytes(body)), tuple(offsets) + + def _build_index( rows: list[dict[str, int | str]], pack_sha1: str, @@ -120,6 +188,261 @@ def test_decodes_real_envelopes_and_verifies_trailer(self) -> None: self.assertEqual(entry.payload_sha256, hashlib.sha256(payload).hexdigest()) self.assertGreater(entry.packed_size, 1) + def test_ofs_codec_matches_independent_git_vectors(self) -> None: + vectors = ( + (1, "01"), + (127, "7f"), + (128, "8000"), + (129, "8001"), + (255, "807f"), + (256, "8100"), + (16_511, "ff7f"), + (16_512, "808000"), + (1_048_576, "beff00"), + ) + for distance, expected_hex in vectors: + encoded = bytes.fromhex(expected_hex) + self.assertEqual(pack_module._encode_ofs_distance(distance), encoded) + decoded, consumed = pack_module._decode_ofs_base_offset( + encoded, + 0, + end=len(encoded), + entry_offset=distance + 12, + ) + self.assertEqual(decoded, 12) + self.assertEqual(consumed, len(encoded)) + + for malformed in (b"\x00", b"\x80", b"\x80\x80\x80\x00"): + with self.subTest(encoded=malformed.hex()), self.assertRaises( + VerificationError + ): + pack_module._decode_ofs_base_offset( + malformed, + 0, + end=len(malformed), + entry_offset=128, + ) + + def test_delta_size_codec_matches_independent_git_vectors(self) -> None: + vectors = ( + (0, "00"), + (127, "7f"), + (128, "8001"), + (16_383, "ff7f"), + (16_384, "808001"), + (262_144, "808010"), + ) + for value, expected_hex in vectors: + encoded = bytes.fromhex(expected_hex) + self.assertEqual(pack_module._encode_delta_size(value), encoded) + decoded, consumed = pack_module._decode_delta_size( + encoded, + 0, + label="fixture size", + ) + self.assertEqual(decoded, value) + self.assertEqual(consumed, len(encoded)) + + for malformed in (b"\x80", b"\x80\x00", b"\x80\x80\x80\x00"): + with self.subTest(encoded=malformed.hex()), self.assertRaises( + VerificationError + ): + pack_module._decode_delta_size( + malformed, + 0, + label="fixture size", + ) + + def test_decodes_bounded_ofs_delta_copy_and_insert(self) -> None: + base = b"bounded base\n" + target = base + b"delta\n" + program = _program( + len(base), + len(target), + _copy_instruction(size=len(base)) + bytes((6,)) + b"delta\n", + ) + content, offsets = _build_ofs_pack(base, ((0, program),)) + + parsed = parse_pack(content) + + self.assertEqual(len(parsed.entries), 2) + delta = parsed.entries[1] + self.assertEqual(delta.representation, "ofs-delta") + self.assertEqual(delta.base_offset, offsets[0]) + self.assertEqual(delta.base_oid, parsed.entries[0].oid) + self.assertEqual(delta.delta_depth, 1) + self.assertEqual(delta.object_type, "blob") + self.assertEqual(delta.size, len(target)) + self.assertEqual(delta.stored_size, len(program)) + self.assertEqual(delta.oid, git_object_oid("blob", target)) + self.assertEqual(delta.payload_sha256, hashlib.sha256(target).hexdigest()) + + def test_resolves_two_deltas_that_share_one_base(self) -> None: + base = b"shared base\n" + first_target = base + b"one\n" + second_target = base + b"two\n" + first_program = _program( + len(base), + len(first_target), + _copy_instruction(size=len(base)) + b"\x04one\n", + ) + second_program = _program( + len(base), + len(second_target), + _copy_instruction(size=len(base)) + b"\x04two\n", + ) + content, offsets = _build_ofs_pack( + base, + ((0, first_program), (0, second_program)), + ) + + parsed = parse_pack(content) + + self.assertEqual(len(parsed.entries), 3) + self.assertEqual(parsed.entries[1].base_offset, offsets[0]) + self.assertEqual(parsed.entries[2].base_offset, offsets[0]) + self.assertEqual(parsed.entries[1].delta_depth, 1) + self.assertEqual(parsed.entries[2].delta_depth, 1) + self.assertEqual(parsed.entries[1].oid, git_object_oid("blob", first_target)) + self.assertEqual(parsed.entries[2].oid, git_object_oid("blob", second_target)) + + def test_decodes_the_default_64k_copy_size(self) -> None: + base = b"a" * 0x10000 + target = base + b"!" + program = _program( + len(base), + len(target), + _copy_instruction(size=0x10000) + b"\x01!", + ) + content, _ = _build_ofs_pack(base, ((0, program),)) + + parsed = parse_pack(content) + + self.assertEqual(parsed.entries[1].size, len(target)) + self.assertEqual(parsed.entries[1].oid, git_object_oid("blob", target)) + + def test_rejects_ref_delta_even_when_its_base_is_present(self) -> None: + base = b"base\n" + target = base + b"ref\n" + program = _program( + len(base), + len(target), + _copy_instruction(size=len(base)) + b"\x04ref\n", + ) + body = bytearray(PACK_MAGIC) + body.extend(PACK_VERSION.to_bytes(4, "big")) + body.extend((2).to_bytes(4, "big")) + body.extend(_entry_header(3, len(base))) + body.extend(zlib.compress(base, level=9)) + body.extend(_entry_header(7, len(program))) + body.extend(bytes.fromhex(git_object_oid("blob", base))) + body.extend(zlib.compress(program, level=9)) + + with self.assertRaisesRegex(VerificationError, "REF_DELTA"): + parse_pack(_resign_pack(bytes(body))) + + def test_rejects_zero_middle_and_underflow_ofs_offsets(self) -> None: + base = b"base\n" + target = base + b"x" + program = _program( + len(base), + len(target), + _copy_instruction(size=len(base)) + b"\x01x", + ) + valid, offsets = _build_ofs_pack(base, ((0, program),)) + header_size = len(_entry_header(6, len(program))) + offset_position = offsets[1] + header_size + distances = ( + b"\x00", + _ofs_distance(offsets[1] - (offsets[0] + 1)), + _ofs_distance(offsets[1] - 11), + ) + for distance in distances: + self.assertEqual(len(distance), 1) + changed = bytearray(valid[:-20]) + changed[offset_position] = distance[0] + with self.subTest(distance=distance.hex()), self.assertRaises( + VerificationError + ): + parse_pack(_resign_pack(bytes(changed))) + + def test_rejects_malformed_delta_programs(self) -> None: + base = b"base" + cases = ( + _program(len(base) + 1, 0, b""), + _program(len(base), MAX_OBJECT_BYTES + 1, b""), + _program(len(base), 1, b"\x00"), + _program(len(base), 2, b"\x02x"), + _program( + len(base), + 1, + _copy_instruction(offset=len(base), size=1), + ), + _program(len(base), 2, b"\x01x"), + _program(len(base), 1, b"\x02xy"), + b"\x00\x00\x00", + b"\x80" * 6, + ) + for program in cases: + with self.subTest(program=program.hex()), self.assertRaises( + VerificationError + ): + pack_module._apply_delta(base, program) + + def test_enforces_depth_instruction_and_aggregate_budgets(self) -> None: + base = b"a" + previous = base + deltas: list[tuple[int, bytes]] = [] + for depth in range(MAX_DELTA_DEPTH + 1): + target = previous + b"x" + deltas.append( + ( + depth, + _program( + len(previous), + len(target), + _copy_instruction(size=len(previous)) + b"\x01x", + ), + ) + ) + previous = target + too_deep, _ = _build_ofs_pack(base, tuple(deltas)) + with self.assertRaisesRegex(VerificationError, "depth"): + parse_pack(too_deep) + + two_inserts = _program(len(base), 2, b"\x01x\x01y") + with ( + mock.patch.object(pack_module, "MAX_DELTA_INSTRUCTIONS", 1), + self.assertRaisesRegex(VerificationError, "instruction"), + ): + pack_module._apply_delta(base, two_inserts) + self.assertEqual(MAX_DELTA_INSTRUCTIONS, 4_096) + + target = base + b"x" + bounded, _ = _build_ofs_pack( + base, + ( + ( + 0, + _program( + len(base), + len(target), + _copy_instruction(size=len(base)) + b"\x01x", + ), + ), + ), + ) + with ( + mock.patch.object( + pack_module, + "MAX_TOTAL_EXPANDED_BYTES", + len(base) + len(target) - 1, + ), + self.assertRaisesRegex(VerificationError, "aggregate"), + ): + parse_pack(bounded) + self.assertEqual(MAX_TOTAL_EXPANDED_BYTES, 16_777_216) + def test_rejects_non_bytes_and_outer_boundary_drift(self) -> None: cases = ( bytearray(self.pack_bytes), @@ -149,6 +472,9 @@ def test_rejects_delta_type_size_zlib_and_trailer_mutations(self) -> None: delta = bytearray(self.pack_bytes[:-20]) delta[12] = (delta[12] & 0x8F) | 0x60 + unsupported_tag = bytearray(self.pack_bytes[:-20]) + unsupported_tag[12] = (unsupported_tag[12] & 0x8F) | 0x40 + wrong_size = bytearray(self.pack_bytes[:-20]) wrong_size[12] = (wrong_size[12] & 0xF0) | ((len(self.payloads[0]) + 1) & 0x0F) @@ -160,6 +486,7 @@ def test_rejects_delta_type_size_zlib_and_trailer_mutations(self) -> None: for content in ( _resign_pack(bytes(delta)), + _resign_pack(bytes(unsupported_tag)), _resign_pack(bytes(wrong_size)), _resign_pack(bytes(corrupt_zlib)), bytes(bad_trailer), @@ -284,6 +611,99 @@ def test_real_pack_and_index_are_deterministic_and_leave_no_files(self) -> None: self.assertFalse(report["scope"]["delta_entries_supported"]) self.assertFalse(report["scope"]["authentication_claim"]) + + def test_real_ofs_pack_is_deterministic_exact_and_leaves_no_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + first = run_ofs_pack_lab(root) + self.assertEqual(list(root.iterdir()), []) + second = run_ofs_pack_lab(root) + self.assertEqual(list(root.iterdir()), []) + + self.assertEqual(first.to_json(), second.to_json()) + report = first.document["report"] + self.assertEqual(report["schema_version"], OFS_PACK_SCHEMA_VERSION) + self.assertEqual(report["pack"]["version"], PACK_VERSION) + self.assertEqual(report["index"]["version"], INDEX_VERSION) + self.assertEqual(report["pack"]["object_count"], 2) + self.assertEqual(report["pack"]["full_count"], 1) + self.assertEqual(report["pack"]["delta_count"], 1) + self.assertEqual(report["pack"]["ofs_delta_count"], 1) + self.assertEqual(report["pack"]["ref_delta_count"], 0) + self.assertEqual(report["pack"]["max_delta_depth"], 1) + self.assertEqual( + report["command_trace"], + ["init", "hash-object", "hash-object", "pack-objects"], + ) + self.assertEqual( + report["pack_objects"]["normalized_argv"], + [ + "git", + "pack-objects", + "--delta-base-offset", + "--window=2", + "--depth=1", + "--threads=1", + "--compression=0", + "--no-reuse-delta", + "--no-reuse-object", + "--index-version=2", + "/fixture", + ], + ) + self.assertEqual(report["pack_objects"]["stdin_bytes"], 82) + self.assertRegex(report["pack_objects"]["stdin_sha256"], r"^[0-9a-f]{64}$") + self.assertTrue(all(report["checks"].values())) + self.assertTrue(report["scope"]["ofs_delta_supported"]) + self.assertFalse(report["scope"]["ref_delta_supported"]) + self.assertFalse(report["scope"]["thin_pack_supported"]) + representations = [ + entry["representation"] for entry in report["objects_in_pack_order"] + ] + self.assertEqual(representations, ["full", "ofs-delta"]) + delta = report["objects_in_pack_order"][1] + self.assertEqual(delta["delta_depth"], 1) + self.assertEqual(delta["base_offset"], report["objects_in_pack_order"][0]["offset"]) + self.assertRegex(delta["ofs_offset_bytes_hex"], r"^[0-9a-f]{2,6}$") + + def test_ofs_pack_cli_exposes_receipt_and_canonical_document(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + stdout = io.StringIO() + stderr = io.StringIO() + self.assertEqual( + main( + ["pack-ofs-verify"], + root=root, + stdout=stdout, + stderr=stderr, + ), + 0, + ) + self.assertEqual(stderr.getvalue(), "") + self.assertRegex( + stdout.getvalue(), + rf"^PASS {OFS_PACK_SCHEMA_VERSION} objects=2 .* " + rf"deltas=1 .* receipt_sha256=[0-9a-f]{{64}}\n$", + ) + + inspect_output = io.StringIO() + self.assertEqual( + main( + ["pack-ofs-inspect", "--compact"], + root=root, + stdout=inspect_output, + stderr=io.StringIO(), + ), + 0, + ) + inspected = json.loads(inspect_output.getvalue()) + self.assertEqual( + inspected["report"]["schema_version"], + OFS_PACK_SCHEMA_VERSION, + ) + self.assertEqual(inspected["report"]["pack"]["ofs_delta_count"], 1) + def test_receipt_binds_the_complete_payload(self) -> None: with tempfile.TemporaryDirectory() as temporary: result = run_pack_lab(Path(temporary)) diff --git a/tools/capture_ofs_report.sh b/tools/capture_ofs_report.sh new file mode 100755 index 0000000..e673d6c --- /dev/null +++ b/tools/capture_ofs_report.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 0 ]]; then + echo "usage: $0" >&2 + exit 2 +fi + +readonly image='mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c' +readonly browser='/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell' +readonly browser_version='Chromium 140.0.7339.186' +readonly browser_sha256='003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d' + +repo_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd -P) +demo_root="$repo_root/docs/demo/git-pack-ofs-delta-v1" +report_path="$demo_root/report.html" +output_path="$repo_root/docs/assets/git-pack-ofs-report.png" +rendered_dom_path="$demo_root/rendered-dom.html" +attestation_path="$demo_root/capture-attestation.json" + +if ! command -v docker >/dev/null 2>&1; then + echo "capture error: Docker is required" >&2 + exit 1 +fi +if [[ $(id -u) -eq 0 || $(id -g) -eq 0 ]]; then + echo "capture error: refusing to run Chromium as root" >&2 + exit 1 +fi + +python3 -B "$repo_root/tools/generate_ofs_evidence.py" --write --allow-missing-screenshot >/dev/null +python3 -B "$repo_root/tools/generate_ofs_evidence.py" --check --allow-missing-screenshot >/dev/null + +for managed_path in \ + "$repo_root/docs" \ + "$repo_root/docs/demo" \ + "$demo_root" \ + "$report_path" \ + "$repo_root/docs/assets"; do + if [[ ! -e "$managed_path" || $(realpath -e -- "$managed_path") != "$managed_path" ]]; then + echo "capture error: managed evidence path is missing or contains a symlink" >&2 + exit 1 + fi +done +if [[ ! -f "$report_path" || -L "$report_path" ]]; then + echo "capture error: offline report is unsafe" >&2 + exit 1 +fi +for destination in "$output_path" "$rendered_dom_path" "$attestation_path"; do + if [[ -L "$destination" || ( -e "$destination" && ! -f "$destination" ) ]]; then + echo "capture error: managed capture destination is unsafe" >&2 + exit 1 + fi + if [[ -e "$destination" && $(stat -c '%h' -- "$destination") -ne 1 ]]; then + echo "capture error: managed capture destination has multiple hard links" >&2 + exit 1 + fi +done + +report_sha_before=$(sha256sum "$report_path" | awk '{print $1}') +script_sha_before=$(sha256sum "$repo_root/tools/capture_ofs_report.sh" | awk '{print $1}') + +observed_digest=$(docker image inspect --format '{{index .RepoDigests 0}}' "$image") +if [[ "$observed_digest" != "$image" ]]; then + echo "capture error: cached container digest is not exact" >&2 + exit 1 +fi +observed_architecture=$(docker image inspect --format '{{.Architecture}}' "$image") +if [[ "$observed_architecture" != "amd64" ]]; then + echo "capture error: cached container architecture is not amd64" >&2 + exit 1 +fi + +uid=$(id -u) +gid=$(id -g) +readonly uid +readonly gid +common=( + --rm + --pull=never + --platform linux/amd64 + --network none + --user "$uid:$gid" + --read-only + --cap-drop ALL + --security-opt no-new-privileges + --pids-limit 256 + --memory 768m + --memory-swap 768m + --cpus 1 + --ulimit nofile=1024:1024 + --ulimit core=0:0 + --tmpfs '/tmp:rw,nosuid,nodev,noexec,size=256m,mode=1777' + --tmpfs '/dev/shm:rw,nosuid,nodev,noexec,size=256m,mode=1777' + --env HOME=/tmp + --env XDG_CACHE_HOME=/tmp/cache + --env XDG_CONFIG_HOME=/tmp/config + --env LANG=C.UTF-8 + --env LC_ALL=C.UTF-8 + --env TZ=UTC +) + +observed_hash=$(docker run "${common[@]}" --entrypoint /usr/bin/sha256sum \ + "$image" "$browser" | awk '{print $1}') +if [[ "$observed_hash" != "$browser_sha256" ]]; then + echo "capture error: Chromium binary hash is not exact" >&2 + exit 1 +fi +observed_version=$(docker run "${common[@]}" --entrypoint "$browser" \ + "$image" --version) +if [[ "$observed_version" != "$browser_version" ]]; then + echo "capture error: Chromium version is not exact" >&2 + exit 1 +fi + +temporary_root=$(mktemp -d "$repo_root/.git-pack-ofs-report-capture.XXXXXX") +cidfile="$temporary_root/container.cid" +cleanup() { + if [[ -f "$cidfile" && ! -L "$cidfile" ]]; then + container_id=$(<"$cidfile") + if [[ "$container_id" =~ ^[0-9a-f]{64}$ ]]; then + docker rm -f "$container_id" >/dev/null 2>&1 || true + fi + fi + if [[ -d "$temporary_root" ]]; then + find "$temporary_root" -depth -delete + fi +} +trap cleanup EXIT INT TERM +mkdir "$temporary_root/output" +dom_path="$temporary_root/rendered-dom.html" +capture_log="$temporary_root/capture.stderr" + +capture=( + docker run + "${common[@]}" + --cidfile "$cidfile" + --mount "type=bind,src=$demo_root,dst=/demo,readonly" + --mount "type=bind,src=$temporary_root/output,dst=/output" + --workdir /demo + --entrypoint "$browser" + "$image" + --headless + --no-sandbox + --disable-background-networking + --disable-breakpad + --disable-component-update + --disable-default-apps + --disable-extensions + '--disable-features=OptimizationHints,Translate' + --disable-sync + --force-color-profile=srgb + --force-device-scale-factor=1 + --hide-scrollbars + '--host-resolver-rules=MAP * ~NOTFOUND' + --lang=en-US + --metrics-recording-only + --no-first-run + --no-pings + --password-store=basic + --run-all-compositor-stages-before-draw + --safebrowsing-disable-auto-update + --virtual-time-budget=1000 + '--window-size=1440,1500' + --dump-dom + --screenshot=/output/git-pack-ofs-report.png + file:///demo/report.html +) + +if ! timeout --signal=TERM --kill-after=5s 45s \ + "${capture[@]}" >"$dom_path" 2>"$capture_log"; then + echo "capture error: isolated Chromium capture failed" >&2 + sed -n '1,12p' "$capture_log" >&2 + exit 1 +fi + +python3 - "$repo_root" "$repo_root/evidence/git-pack-ofs-delta-v1.json" "$dom_path" "$temporary_root/output" <<'PY' +from pathlib import Path +import json +import sys + +repo_root = Path(sys.argv[1]) +evidence_path = Path(sys.argv[2]) +dom_path = Path(sys.argv[3]) +output = Path(sys.argv[4]) +sys.path.insert(0, str(repo_root)) +from tools.generate_pack_evidence import _parse_pack_png + +document = json.loads(evidence_path.read_text(encoding="utf-8")) +receipt = document["receipt"]["sha256"] +dom = dom_path.read_text(encoding="utf-8") +required = ( + '
/dev/null +if [[ $(sha256sum "$report_path" | awk '{print $1}') != "$report_sha_before" ]]; then + echo "capture error: offline report changed during capture" >&2 + exit 1 +fi +if [[ $(sha256sum "$repo_root/tools/capture_ofs_report.sh" | awk '{print $1}') != "$script_sha_before" ]]; then + echo "capture error: capture script changed during execution" >&2 + exit 1 +fi + +chmod 0600 "$temporary_root/output/git-pack-ofs-report.png" +chmod 0600 "$dom_path" +mv -f -- "$temporary_root/output/git-pack-ofs-report.png" "$output_path" +mv -f -- "$dom_path" "$rendered_dom_path" + +python3 - "$repo_root" "$report_path" "$rendered_dom_path" "$output_path" "$attestation_path" <<'PY' +from pathlib import Path +import hashlib +import json +import os +import sys +import tempfile + +root = Path(sys.argv[1]) +report_path = Path(sys.argv[2]) +dom_path = Path(sys.argv[3]) +screenshot_path = Path(sys.argv[4]) +attestation_path = Path(sys.argv[5]) + +def sha256(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + +def row(path: Path, relative: str) -> dict[str, object]: + content = path.read_bytes() + return {"path": relative, "sha256": sha256(content), "size": len(content)} + +report = report_path.read_bytes() +dom = dom_path.read_bytes() +screenshot = screenshot_path.read_bytes() +script_path = root / "tools/capture_ofs_report.sh" +script = script_path.read_bytes() +evidence = json.loads((root / "evidence/git-pack-ofs-delta-v1.json").read_text(encoding="utf-8")) + +payload = { + "browser": { + "binary_path": "/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell", + "sha256": "003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d", + "version": "Chromium 140.0.7339.186", + }, + "container": { + "architecture": "amd64", + "image": "mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c", + }, + "input": { + "report": { + **row(report_path, "docs/demo/git-pack-ofs-delta-v1/report.html"), + "report_receipt_sha256": evidence["receipt"]["sha256"], + } + }, + "isolation": { + "capabilities": "all-dropped", + "demo_mount": "read-only", + "network": "none", + "no_new_privileges": True, + "pull": "never", + "root_filesystem": "read-only", + "user": "caller-nonroot", + }, + "outputs": { + "rendered_dom": row(dom_path, "docs/demo/git-pack-ofs-delta-v1/rendered-dom.html"), + "screenshot": { + **row(screenshot_path, "docs/assets/git-pack-ofs-report.png"), + "height": 1500, + "width": 1440, + }, + }, + "schema_version": "git-pack-ofs-browser-capture-attestation/v1", + "script": row(script_path, "tools/capture_ofs_report.sh"), + "viewport": {"device_scale_factor": 1, "height": 1500, "width": 1440}, +} +canonical = json.dumps( + payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True +).encode("utf-8") +document = { + "attestation": payload, + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": sha256(canonical), + }, +} +content = (json.dumps(document, ensure_ascii=True, indent=2, sort_keys=True) + "\n").encode("utf-8") +temporary_path: Path | None = None +try: + with tempfile.NamedTemporaryFile( + mode="wb", prefix=".capture-attestation-", dir=attestation_path.parent, delete=False + ) as temporary: + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + temporary_path = Path(temporary.name) + os.chmod(temporary_path, 0o600) + os.replace(temporary_path, attestation_path) + temporary_path = None +finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) +PY + +python3 -B "$repo_root/tools/generate_ofs_evidence.py" --write >/dev/null +python3 -B "$repo_root/tools/generate_ofs_evidence.py" --check >/dev/null +sha256sum "$output_path" diff --git a/tools/generate_ofs_evidence.py b/tools/generate_ofs_evidence.py new file mode 100755 index 0000000..9abd42a --- /dev/null +++ b/tools/generate_ofs_evidence.py @@ -0,0 +1,922 @@ +#!/usr/bin/env python3 +"""Generate and verify source-bound evidence for the real OFS_DELTA fixture.""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +import hashlib +import html +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import sys +from typing import Any + +ROOT = Path(__file__).resolve().parents[1] +if os.fspath(ROOT) not in sys.path: + sys.path.insert(0, os.fspath(ROOT)) + +from tools.generate_evidence import ( + BROWSER_PATH, + BROWSER_SHA256, + BROWSER_VERSION, + CONTAINER_IMAGE, + EvidenceError, + _artifact_row, + _check_artifacts, + _json_bytes, + _safe_capture_bytes, + _sha256, + _source_row, + _svg_header as _base_svg_header, + _write_artifacts, + _xml_text, +) +from tools.generate_pack_evidence import _parse_pack_png + + +EVIDENCE_PATH = Path("evidence/git-pack-ofs-delta-v1.json") +DEMO_ROOT = Path("docs/demo/git-pack-ofs-delta-v1") +VERIFY_PATH = DEMO_ROOT / "verify.txt" +INSPECT_PATH = DEMO_ROOT / "inspect.json" +REPORT_PATH = DEMO_ROOT / "report.html" +MANIFEST_PATH = DEMO_ROOT / "manifest.json" +RENDERED_DOM_PATH = DEMO_ROOT / "rendered-dom.html" +ATTESTATION_PATH = DEMO_ROOT / "capture-attestation.json" +RECONSTRUCTION_PATH = Path("docs/assets/git-pack-ofs-reconstruction.svg") +WORKFLOW_PATH = Path("docs/assets/git-pack-ofs-workflow.svg") +CLI_PATH = Path("docs/assets/git-pack-ofs-cli.svg") +SCREENSHOT_PATH = Path("docs/assets/git-pack-ofs-report.png") +PNG_DIMENSIONS = (1440, 1500) +SHA1_RE = re.compile(r"^[0-9a-f]{40}$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") +HEX_BYTES_RE = re.compile(r"^(?:[0-9a-f]{2}){1,3}$") +EXPECTED_ARGV = [ + "git", + "pack-objects", + "--delta-base-offset", + "--window=2", + "--depth=1", + "--threads=1", + "--compression=0", + "--no-reuse-delta", + "--no-reuse-object", + "--index-version=2", + "/fixture", +] +EXPECTED_CHECKS = { + "all_fixture_objects_present", + "exactly_one_full_entry", + "exactly_one_ofs_delta_entry", + "index_checksum_verified", + "index_crc32_matches_pack", + "index_fanout_matches_sorted_oids", + "index_offsets_match_pack", + "ofs_base_entry_bound", + "ofs_distance_reencoded", + "pack_trailer_verified", + "reconstructed_objects_match_fixture", + "ref_delta_entries_absent", +} + + +@dataclass(frozen=True, slots=True) +class OfsCapture: + screenshot: bytes + rendered_dom: bytes + attestation: bytes + document: Mapping[str, Any] + + +def _ofs_svg_header( + title: str, + description: str, + *, + width: int, + height: int, + receipt: str, + metadata_extra: Mapping[str, Any] | None = None, +) -> str: + metadata = dict(metadata_extra or {}) + metadata["source"] = EVIDENCE_PATH.as_posix() + return _base_svg_header( + title, + description, + width=width, + height=height, + receipt=receipt, + metadata_extra=metadata, + ) + + +def _cli_environment() -> dict[str, str]: + return { + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": os.environ.get("PATH", ""), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + "TZ": "UTC", + } + + +def _run_cli(arguments: Sequence[str]) -> bytes: + completed = subprocess.run( + [sys.executable, "-B", "-m", "git_dag_lab", *arguments], + cwd=ROOT, + env=_cli_environment(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + shell=False, + timeout=30, + ) + if completed.returncode != 0: + raise EvidenceError(f"OFS CLI {' '.join(arguments)} failed") + if completed.stderr: + raise EvidenceError(f"OFS CLI {' '.join(arguments)} wrote to stderr") + return completed.stdout + + +def _collect_cli() -> tuple[bytes, bytes, bytes, dict[str, Any]]: + first = ( + _run_cli(("pack-ofs-verify",)), + _run_cli(("pack-ofs-inspect", "--compact")), + _run_cli(("pack-ofs-inspect",)), + ) + second = ( + _run_cli(("pack-ofs-verify",)), + _run_cli(("pack-ofs-inspect", "--compact")), + _run_cli(("pack-ofs-inspect",)), + ) + if first != second: + raise EvidenceError("two fresh OFS CLI runs produced different evidence") + verify_output, compact_output, pretty_output = first + try: + compact_document = json.loads(compact_output) + pretty_document = json.loads(pretty_output) + except json.JSONDecodeError as exc: + raise EvidenceError("OFS inspection output is not JSON") from exc + if compact_document != pretty_document: + raise EvidenceError("compact and pretty OFS documents differ") + if compact_output != _json_bytes(compact_document, pretty=False): + raise EvidenceError("compact OFS inspection output is not canonical JSON") + if pretty_output != _json_bytes(pretty_document, pretty=True): + raise EvidenceError("pretty OFS inspection output is not canonical JSON") + _validate_document(compact_document, verify_output) + return verify_output, compact_output, pretty_output, compact_document + + +def _validate_document(document: Mapping[str, Any], verify_output: bytes) -> None: + if set(document) != {"receipt", "report"}: + raise EvidenceError("OFS evidence has unexpected top-level fields") + report = document.get("report") + receipt = document.get("receipt") + if not isinstance(report, dict) or not isinstance(receipt, dict): + raise EvidenceError("OFS evidence sections are malformed") + if report.get("schema_version") != "git-pack-ofs-delta-lab/v1": + raise EvidenceError("unexpected OFS evidence schema") + + receipt_sha = receipt.get("sha256") + if not isinstance(receipt_sha, str) or not SHA256_RE.fullmatch(receipt_sha): + raise EvidenceError("OFS receipt is malformed") + canonical = json.dumps( + report, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + if _sha256(canonical) != receipt_sha: + raise EvidenceError("OFS receipt does not bind the canonical report") + + pack = report.get("pack", {}) + index = report.get("index", {}) + checks = report.get("checks", {}) + scope = report.get("scope", {}) + fixture = report.get("fixture", {}) + objects = report.get("objects_in_pack_order", []) + pack_objects = report.get("pack_objects", {}) + if ( + pack.get("version") != 2 + or pack.get("object_count") != 2 + or pack.get("full_count") != 1 + or pack.get("delta_count") != 1 + or pack.get("ofs_delta_count") != 1 + or pack.get("ref_delta_count") != 0 + or pack.get("max_delta_depth") != 1 + or not isinstance(pack.get("bytes"), int) + or not 32 < pack["bytes"] <= 1_048_576 + or not SHA1_RE.fullmatch(pack.get("trailer_sha1", "")) + or not SHA256_RE.fullmatch(pack.get("sha256", "")) + ): + raise EvidenceError("OFS pack summary is outside the reviewed fixture") + if ( + index.get("version") != 2 + or index.get("pack_sha1") != pack["trailer_sha1"] + or not isinstance(index.get("bytes"), int) + or not 1_064 < index["bytes"] <= 1_048_576 + or not SHA1_RE.fullmatch(index.get("index_sha1", "")) + or not SHA256_RE.fullmatch(index.get("sha256", "")) + ): + raise EvidenceError("OFS index summary is outside the reviewed fixture") + if set(checks) != EXPECTED_CHECKS or not all( + value is True for value in checks.values() + ): + raise EvidenceError("not every OFS pack/index check passed") + + mutation = fixture.get("changed_record", {}) + fixture_rows = fixture.get("objects", []) + if ( + fixture.get("object_count") != 2 + or len(fixture_rows) != 2 + or len(objects) != 2 + or mutation + != { + "after": "1024:fedcba9876543210fedcba9876543210", + "before": "1024:0123456789abcdef0123456789abcdef", + "line_number": 1024, + } + ): + raise EvidenceError("OFS fixture inventory or mutation is not exact") + fixture_by_oid: dict[str, Mapping[str, Any]] = {} + for row in fixture_rows: + oid = row.get("oid", "") + if ( + not SHA1_RE.fullmatch(oid) + or not SHA256_RE.fullmatch(row.get("payload_sha256", "")) + or row.get("size") != 77_824 + or row.get("label") not in {"baseline", "line-1024-changed"} + ): + raise EvidenceError("OFS fixture row is malformed") + fixture_by_oid[oid] = row + if len(fixture_by_oid) != 2: + raise EvidenceError("OFS fixture object IDs are not unique") + + full, delta = objects + for row in objects: + fixture_row = fixture_by_oid.get(row.get("oid", "")) + if ( + fixture_row is None + or row.get("object_type") != "blob" + or row.get("size") != 77_824 + or row.get("size") != fixture_row.get("size") + or row.get("payload_sha256") != fixture_row.get("payload_sha256") + or row.get("label") != fixture_row.get("label") + or row.get("offset") != row.get("index_offset") + or row.get("crc32") != row.get("index_crc32") + or not isinstance(row.get("packed_size"), int) + or row["packed_size"] < 1 + ): + raise EvidenceError("OFS object row is not fixture- and index-bound") + if ( + full.get("representation") != "full" + or delta.get("representation") != "ofs-delta" + or delta.get("base_offset") != full.get("offset") + or delta.get("base_oid") != full.get("oid") + or delta.get("delta_depth") != 1 + or not isinstance(delta.get("stored_size"), int) + or not 4 <= delta["stored_size"] <= 262_144 + or not isinstance(delta.get("ofs_distance"), int) + or delta["ofs_distance"] != delta["offset"] - full["offset"] + or delta["ofs_distance"] < 1 + or not HEX_BYTES_RE.fullmatch(delta.get("ofs_offset_bytes_hex", "")) + ): + raise EvidenceError("OFS physical base/delta relationship is incomplete") + + oid_order = pack_objects.get("stdin_oid_order", []) + if ( + pack_objects.get("normalized_argv") != EXPECTED_ARGV + or pack_objects.get("stdin_bytes") != 82 + or oid_order != sorted(fixture_by_oid) + or not SHA256_RE.fullmatch(pack_objects.get("stdin_sha256", "")) + or report.get("command_trace") + != ["init", "hash-object", "hash-object", "pack-objects"] + ): + raise EvidenceError("OFS command provenance is incomplete") + if ( + scope.get("arbitrary_repository_supported") is not False + or scope.get("authentication_claim") is not False + or scope.get("byte_identity_requires_same_git_build") is not True + or scope.get("git_pack_objects_executed") is not True + or scope.get("network_required") is not False + or scope.get("ofs_delta_supported") is not True + or scope.get("ref_delta_supported") is not False + or scope.get("thin_pack_supported") is not False + ): + raise EvidenceError("OFS scope and non-claims are incomplete") + + expected_verify = ( + f"PASS git-pack-ofs-delta-lab/v1 objects=2 pack_version=2 " + f"index_version=2 deltas=1 pack_sha1={pack['trailer_sha1']} " + f"receipt_sha256={receipt_sha}\n" + ).encode("ascii") + if verify_output != expected_verify: + raise EvidenceError("OFS verification transcript is not exact") + rendered = json.dumps(document, ensure_ascii=True, sort_keys=True) + forbidden = ("/home/", "github.com/", "@gmail.com", "AKIA", "ghp_", "github_pat_") + if any(marker.lower() in rendered.lower() for marker in forbidden): + raise EvidenceError("OFS evidence contains host, remote, or credential material") + + +def _bounded_binary_hash(path: Path, *, label: str) -> str: + try: + resolved = path.resolve(strict=True) + metadata = resolved.stat() + except (OSError, RuntimeError) as exc: + raise EvidenceError(f"{label} is unavailable") from exc + if ( + not stat.S_ISREG(metadata.st_mode) + or metadata.st_size < 1 + or metadata.st_size > 100 * 1024 * 1024 + ): + raise EvidenceError(f"{label} is outside the evidence bound") + try: + return _sha256(resolved.read_bytes()) + except OSError as exc: + raise EvidenceError(f"{label} could not be read") from exc + + +def _git_environment_proof() -> dict[str, object]: + candidate = shutil.which("git", path=_cli_environment()["PATH"]) + if candidate is None: + raise EvidenceError("Git executable is unavailable for OFS provenance") + git = Path(candidate).resolve(strict=True) + version = subprocess.run( + [os.fspath(git), "version"], + cwd=ROOT, + env=_cli_environment(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + shell=False, + timeout=10, + ) + exec_path = subprocess.run( + [os.fspath(git), "--exec-path"], + cwd=ROOT, + env=_cli_environment(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + shell=False, + timeout=10, + ) + if ( + version.returncode != 0 + or version.stderr + or exec_path.returncode != 0 + or exec_path.stderr + ): + raise EvidenceError("Git build provenance commands failed") + try: + version_text = version.stdout.decode("ascii").strip() + exec_path_text = exec_path.stdout.decode("utf-8").strip() + except UnicodeDecodeError as exc: + raise EvidenceError("Git build provenance output is malformed") from exc + if not re.fullmatch(r"git version [0-9]+(?:\.[0-9]+){1,3}", version_text): + raise EvidenceError("Git version string is outside the reviewed form") + pack_objects = Path(exec_path_text) / "git-pack-objects" + return { + "byte_identity_scope": "same recorded Git build", + "frontend_sha256": _bounded_binary_hash(git, label="Git frontend"), + "pack_objects_sha256": _bounded_binary_hash( + pack_objects, + label="Git pack-objects executable", + ), + "version": version_text, + } + + +def _reconstruction_svg(document: Mapping[str, Any]) -> bytes: + report = document["report"] + receipt = document["receipt"]["sha256"] + pack = report["pack"] + full, delta = report["objects_in_pack_order"] + parts = [ + _ofs_svg_header( + "Real OFS_DELTA reconstruction", + "Physical offsets, the biased backward distance, logical object sizes, and reconstructed object IDs come from the verified Git-generated pack.", + width=1440, + height=760, + receipt=receipt, + metadata_extra={ + "base_oid": full["oid"], + "delta_oid": delta["oid"], + "pack_sha256": pack["sha256"], + }, + ), + ' REAL GIT PACK · ONE FULL BLOB · ONE OFS_DELTA · ZERO REF_DELTA', + ' A physical backward edge reconstructs a new logical blob.', + ' ', + ' BASE ENTRY · FULL', + f' offset {full["offset"]}', + f' {full["packed_size"]} packed bytes · {full["size"]} logical bytes', + f' {_xml_text(full["label"])}', + f' {full["oid"]}', + ' ', + ' DELTA ENTRY · TYPE 6', + f' offset {delta["offset"]} − distance {delta["ofs_distance"]}', + f' biased bytes 0x{delta["ofs_offset_bytes_hex"]} · program {delta["stored_size"]} bytes', + f' {_xml_text(delta["label"])} · depth {delta["delta_depth"]}', + f' {delta["oid"]}', + ' ', + f' base_offset = {delta["base_offset"]}', + ' ', + f' INDEPENDENT REPLAY · {len(report["checks"])}/{len(report["checks"])} CHECKS', + f' Base OID {full["oid"]} → apply bounded copy/insert program → target OID {delta["oid"]}', + f' Index v2 binds physical offsets {full["index_offset"]} and {delta["index_offset"]}, plus CRC32 {full["crc32"]} and {delta["crc32"]}.', + f' receipt {receipt}', + "\n", + ] + return "\n".join(parts).encode("utf-8") + + +def _workflow_svg( + document: Mapping[str, Any], + git_environment: Mapping[str, object], +) -> bytes: + report = document["report"] + receipt = document["receipt"]["sha256"] + command = report["pack_objects"]["normalized_argv"] + stages = ( + ("FIXTURE", "2 × 77,824 B"), + ("REAL GIT", git_environment["version"]), + ("PACK PARSER", "OFS replay"), + ("INDEX V2", "offset + CRC"), + ("RECEIPT", receipt[:16] + "…"), + ) + parts = [ + _ofs_svg_header( + "Reproducible OFS evidence workflow", + "The exact fixed inputs, normalized Git arguments, independent parser, index cross-check, and receipt shown here are bound into the evidence manifest.", + width=1440, + height=690, + receipt=receipt, + metadata_extra={ + "git_frontend_sha256": git_environment["frontend_sha256"], + "stdin_sha256": report["pack_objects"]["stdin_sha256"], + }, + ), + ' FIXED INPUTS · SINGLE THREAD · SAME RECORDED GIT BUILD · FRESH RUNS ×2', + ' From two synthetic blobs to one source-bound receipt.', + ] + x_values = (35, 315, 595, 875, 1155) + for number, ((title, detail), x) in enumerate(zip(stages, x_values, strict=True), 1): + visible = str(detail) if len(str(detail)) <= 24 else str(detail)[:20] + "…" + parts.extend( + ( + f' ', + f' 0{number}', + f' {title}', + f' {_xml_text(visible)}', + f' PASS', + ) + ) + if number < len(stages): + parts.append(f' ') + first_line = " ".join(command[:6]) + second_line = " ".join(command[6:]) + parts.extend( + ( + ' ', + f' $ {_xml_text(first_line)}', + f' {_xml_text(second_line)}', + f' stdin SHA-256 {report["pack_objects"]["stdin_sha256"]}', + f' Git frontend SHA-256 {git_environment["frontend_sha256"]} · byte identity scope: same recorded build', + "\n", + ) + ) + return "\n".join(parts).encode("utf-8") + + +def _cli_svg(document: Mapping[str, Any], verify_output: bytes) -> bytes: + receipt = document["receipt"]["sha256"] + tokens = verify_output.decode("ascii").strip().split() + lines = ( + " ".join(tokens[:5]), + " ".join(tokens[5:7]), + tokens[7], + ) + parts = [ + _ofs_svg_header( + "Real OFS_DELTA CLI receipt", + "Exact stdout from pack-ofs-verify after real git pack-objects output was reconstructed and cross-checked against index v2.", + width=1440, + height=560, + receipt=receipt, + ), + ' REAL CLI STDOUT · STDERR 0 BYTES · EXIT 0', + ' ', + ' ', + ' $ python3 -B -m git_dag_lab pack-ofs-verify', + f' {_xml_text(lines[0])}', + f' {_xml_text(lines[1])}', + f' {_xml_text(lines[2])}', + f' receipt-bound source · {EVIDENCE_PATH.as_posix()}', + f' receipt {receipt}', + "\n", + ] + return "\n".join(parts).encode("utf-8") + + +def _report_html( + document: Mapping[str, Any], + verify_output: bytes, + git_environment: Mapping[str, object], +) -> bytes: + report = document["report"] + receipt = document["receipt"]["sha256"] + pack = report["pack"] + index = report["index"] + checks = report["checks"] + full, delta = report["objects_in_pack_order"] + transcript = html.escape(verify_output.decode("ascii").rstrip("\n")) + object_rows = "".join( + f'''{html.escape(row["representation"])}{html.escape(row["label"])}{row["oid"]}{row["offset"]}{row["stored_size"] if row["representation"] == "ofs-delta" else row["size"]}{row["crc32"]}''' + for row in report["objects_in_pack_order"] + ) + check_rows = "".join( + f"
  • ✓{html.escape(name.replace('_', ' '))}
    • " + for name, passed in checks.items() + if passed + ) + command = html.escape(" ".join(report["pack_objects"]["normalized_argv"])) + text = f''' + + + + + + Git OFS_DELTA evidence — verified offline report + + + +
    +

    Git pack lab · actual OFS_DELTA offline report

    +

    One backward edge.
    Two verified blobs.

    +

    Real git pack-objects --delta-base-offset output is decoded without asking Git to replay it: the bounded delta program reconstructs the target, then index v2 independently binds both physical offsets and CRC32 rows.

    +
    PASS · report receipt {receipt}
    +
    +
    2logical blobs
    +
    1physical OFS_DELTA
    +
    {pack["bytes"]}actual pack bytes
    +
    12/12checks passed
    +
    +
    +

    Objects in physical pack order

    {object_rows}
    formfixtureOIDoffsetstoredCRC32
    Base → target
    offset {full["offset"]} → {delta["offset"]}; biased distance {delta["ofs_distance"]} encoded as {delta["ofs_offset_bytes_hex"]}
    {full["oid"]}
    ↓ independent copy/insert replay
    {delta["oid"]}
    +

    Independent fail-closed gate

      {check_rows}
    +
    +

    Exact command provenance

    $ {command}
+stdin sha256={report["pack_objects"]["stdin_sha256"]}
+Git={html.escape(str(git_environment["version"]))}
+frontend sha256={git_environment["frontend_sha256"]}

    Real CLI receipt

    $ python3 -B -m git_dag_lab pack-ofs-verify
+{transcript}
    Index checksum {index["index_sha1"]} · REF_DELTA false · thin pack false · arbitrary repository false · authentication claim false · network required false · exact byte identity scoped to the recorded Git build.
    +
    Generated offline · no JavaScript · no external assets{EVIDENCE_PATH.as_posix()}
    +
    + + +''' + return text.encode("utf-8") + + +def _capture_path_is_safe_if_present(path: Path) -> None: + target = ROOT / path + if target.exists() and (not target.is_file() or target.is_symlink()): + raise EvidenceError(f"unsafe OFS capture path: {path.as_posix()}") + + +def _load_capture( + report_content: bytes, + receipt: str, + *, + allow_missing: bool, +) -> OfsCapture | None: + for path in (SCREENSHOT_PATH, RENDERED_DOM_PATH, ATTESTATION_PATH): + _capture_path_is_safe_if_present(path) + if allow_missing: + return None + present = [ + (ROOT / path).is_file() + for path in (SCREENSHOT_PATH, RENDERED_DOM_PATH, ATTESTATION_PATH) + ] + if not all(present): + raise EvidenceError("OFS browser capture is missing") + + screenshot = _safe_capture_bytes(SCREENSHOT_PATH, "OFS screenshot") + rendered_dom = _safe_capture_bytes(RENDERED_DOM_PATH, "OFS rendered DOM") + attestation = _safe_capture_bytes(ATTESTATION_PATH, "OFS capture attestation") + if _parse_pack_png(screenshot) != PNG_DIMENSIONS: + raise EvidenceError("OFS screenshot dimensions differ from the contract") + try: + dom_text = rendered_dom.decode("utf-8") + document = json.loads(attestation) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvidenceError("OFS browser capture metadata is malformed") from exc + required = ( + f'data-ofs-receipt="{receipt}"', + 'data-object-count="2"', + 'data-check-count="12"', + "actual OFS_DELTA", + "Base → target", + "Two verified blobs", + ) + if not all(marker in dom_text for marker in required): + raise EvidenceError("OFS rendered DOM sentinels are incomplete") + if any(marker in dom_text for marker in ("/home/", "ERR_FILE", "github.com/")): + raise EvidenceError("OFS rendered DOM contains a forbidden marker") + if set(document) != {"attestation", "receipt"}: + raise EvidenceError("OFS capture attestation shape is invalid") + payload = document["attestation"] + attestation_receipt = document["receipt"].get("sha256") + canonical = json.dumps( + payload, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + if attestation_receipt != _sha256(canonical): + raise EvidenceError("OFS capture attestation receipt is invalid") + try: + report_row = payload["input"]["report"] + dom_row = payload["outputs"]["rendered_dom"] + screenshot_row = payload["outputs"]["screenshot"] + script_row = payload["script"] + except (KeyError, TypeError) as exc: + raise EvidenceError("OFS capture attestation rows are incomplete") from exc + script_path = Path("tools/capture_ofs_report.sh") + script_content = _safe_capture_bytes(script_path, "OFS capture script") + if ( + payload.get("schema_version") + != "git-pack-ofs-browser-capture-attestation/v1" + or payload.get("browser", {}).get("binary_path") != BROWSER_PATH + or payload.get("browser", {}).get("sha256") != BROWSER_SHA256 + or payload.get("browser", {}).get("version") != BROWSER_VERSION + or payload.get("container", {}).get("image") != CONTAINER_IMAGE + or payload.get("container", {}).get("architecture") != "amd64" + or payload.get("isolation", {}).get("network") != "none" + or payload.get("isolation", {}).get("root_filesystem") != "read-only" + or payload.get("isolation", {}).get("demo_mount") != "read-only" + or payload.get("isolation", {}).get("capabilities") != "all-dropped" + or payload.get("isolation", {}).get("no_new_privileges") is not True + or payload.get("viewport") + != { + "device_scale_factor": 1, + "height": PNG_DIMENSIONS[1], + "width": PNG_DIMENSIONS[0], + } + or report_row.get("path") != REPORT_PATH.as_posix() + or report_row.get("sha256") != _sha256(report_content) + or report_row.get("size") != len(report_content) + or report_row.get("report_receipt_sha256") != receipt + or dom_row.get("path") != RENDERED_DOM_PATH.as_posix() + or dom_row.get("sha256") != _sha256(rendered_dom) + or dom_row.get("size") != len(rendered_dom) + or screenshot_row.get("path") != SCREENSHOT_PATH.as_posix() + or screenshot_row.get("sha256") != _sha256(screenshot) + or screenshot_row.get("size") != len(screenshot) + or screenshot_row.get("width") != PNG_DIMENSIONS[0] + or screenshot_row.get("height") != PNG_DIMENSIONS[1] + or script_row.get("path") != script_path.as_posix() + or script_row.get("sha256") != _sha256(script_content) + or script_row.get("size") != len(script_content) + ): + raise EvidenceError("OFS capture attestation does not bind current inputs") + return OfsCapture( + screenshot=screenshot, + rendered_dom=rendered_dom, + attestation=attestation, + document=document, + ) + + +def build_artifacts(*, allow_missing_screenshot: bool) -> dict[Path, bytes]: + verify_output, compact_output, pretty_output, document = _collect_cli() + git_environment = _git_environment_proof() + report_content = _report_html(document, verify_output, git_environment) + receipt = document["receipt"]["sha256"] + capture = _load_capture( + report_content, + receipt, + allow_missing=allow_missing_screenshot, + ) + generated = { + EVIDENCE_PATH: compact_output, + VERIFY_PATH: verify_output, + INSPECT_PATH: pretty_output, + REPORT_PATH: report_content, + RECONSTRUCTION_PATH: _reconstruction_svg(document), + WORKFLOW_PATH: _workflow_svg(document, git_environment), + CLI_PATH: _cli_svg(document, verify_output), + } + roles = { + EVIDENCE_PATH: "canonical compact real OFS_DELTA CLI evidence", + VERIFY_PATH: "exact pack-ofs-verify stdout", + INSPECT_PATH: "exact pretty pack-ofs-inspect stdout", + REPORT_PATH: "dependency-free OFS_DELTA offline report", + RECONSTRUCTION_PATH: "actual OFS base distance and reconstructed object flow", + WORKFLOW_PATH: "source-bound Git OFS evidence workflow", + CLI_PATH: "visualized exact pack-ofs-verify transcript", + } + artifact_rows = [ + _artifact_row(path, generated[path], roles[path]) for path in sorted(generated) + ] + capture_manifest: dict[str, Any] = { + "attestation": ATTESTATION_PATH.as_posix(), + "browser_binary": BROWSER_PATH, + "browser_sha256": BROWSER_SHA256, + "browser_version": BROWSER_VERSION, + "container_image": CONTAINER_IMAGE, + "network": "none", + "status": "not-attested", + "viewport": {"height": PNG_DIMENSIONS[1], "width": PNG_DIMENSIONS[0]}, + } + if capture is not None: + artifact_rows.extend( + ( + _artifact_row( + SCREENSHOT_PATH, + capture.screenshot, + "attested OFS_DELTA offline report browser capture", + ), + _artifact_row( + RENDERED_DOM_PATH, + capture.rendered_dom, + "actual DOM emitted during the OFS report capture", + ), + _artifact_row( + ATTESTATION_PATH, + capture.attestation, + "OFS capture provenance and isolation attestation", + ), + ) + ) + capture_manifest.update( + { + "attestation_receipt_sha256": capture.document["receipt"]["sha256"], + "rendered_dom_sha256": _sha256(capture.rendered_dom), + "screenshot_sha256": _sha256(capture.screenshot), + "status": "attested", + } + ) + artifact_rows.sort(key=lambda row: row["path"]) + manifest = { + "artifacts": artifact_rows, + "capture": capture_manifest, + "commands": [ + { + "argv": ["python3", "-B", "-m", "git_dag_lab", "pack-ofs-verify"], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": VERIFY_PATH.as_posix(), + }, + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-ofs-inspect", + "--compact", + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": EVIDENCE_PATH.as_posix(), + }, + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-ofs-inspect", + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": INSPECT_PATH.as_posix(), + }, + ], + "git_build": git_environment, + "report_receipt_sha256": receipt, + "schema_version": "git-pack-ofs-evidence-manifest/v1", + "sources": [ + _source_row(Path("git_dag_lab/__init__.py")), + _source_row(Path("git_dag_lab/__main__.py")), + _source_row(Path("git_dag_lab/cli.py")), + _source_row(Path("git_dag_lab/lab.py")), + _source_row(Path("git_dag_lab/pack.py")), + _source_row(Path("tools/generate_evidence.py")), + _source_row(Path("tools/generate_pack_evidence.py")), + _source_row(Path("tools/generate_ofs_evidence.py")), + _source_row(Path("tools/capture_ofs_report.sh")), + ], + } + generated[MANIFEST_PATH] = _json_bytes(manifest, pretty=True) + _validate_generated(generated, capture, allow_missing_screenshot) + return generated + + +def _validate_generated( + generated: Mapping[Path, bytes], + capture: OfsCapture | None, + allow_missing_screenshot: bool, +) -> None: + manifest = json.loads(generated[MANIFEST_PATH]) + rows = {row["path"]: row for row in manifest["artifacts"]} + for path, content in generated.items(): + if path == MANIFEST_PATH: + continue + row = rows.get(path.as_posix()) + if ( + row is None + or row["sha256"] != _sha256(content) + or row["size"] != len(content) + ): + raise EvidenceError(f"OFS manifest does not bind {path.as_posix()}") + if capture is not None: + external = { + SCREENSHOT_PATH: capture.screenshot, + RENDERED_DOM_PATH: capture.rendered_dom, + ATTESTATION_PATH: capture.attestation, + } + for path, content in external.items(): + row = rows.get(path.as_posix()) + if ( + row is None + or row["sha256"] != _sha256(content) + or row["size"] != len(content) + ): + raise EvidenceError(f"OFS manifest does not bind {path.as_posix()}") + if manifest["capture"].get("status") != "attested": + raise EvidenceError("OFS manifest omits its attested capture") + elif not allow_missing_screenshot: + raise EvidenceError("OFS manifest has no required browser capture") + + receipt = manifest["report_receipt_sha256"] + for path in (RECONSTRUCTION_PATH, WORKFLOW_PATH, CLI_PATH, REPORT_PATH): + text = generated[path].decode("utf-8") + if receipt not in text: + raise EvidenceError(f"{path.as_posix()} is not receipt-bound") + without_namespace = text.replace("http://www.w3.org/2000/svg", "") + if ( + "https://" in without_namespace + or "http://" in without_namespace + or "/home/" in without_namespace + ): + raise EvidenceError(f"{path.as_posix()} contains an external reference") + for path in (RECONSTRUCTION_PATH, WORKFLOW_PATH, CLI_PATH): + text = generated[path].decode("utf-8") + if " argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--write", action="store_true", help="write generated evidence") + action.add_argument("--check", action="store_true", help="verify checked-in evidence") + parser.add_argument( + "--allow-missing-screenshot", + action="store_true", + help="permit the pre-capture package used by capture_ofs_report.sh", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + generated = build_artifacts( + allow_missing_screenshot=args.allow_missing_screenshot + ) + if args.write: + _write_artifacts(generated) + else: + _check_artifacts(generated) + except EvidenceError as exc: + print(f"OFS evidence error: {exc}", file=sys.stderr) + return 1 + action = "wrote" if args.write else "verified" + screenshot = "optional" if args.allow_missing_screenshot else "required" + print( + f"PASS ofs-evidence {action} artifacts={len(generated)} " + f"screenshot={screenshot}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())