diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25eea94..29dd7ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,10 @@ jobs: cmp docs/demo/git-dag-v1/verify.txt "$RUNNER_TEMP/verify.txt" env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab inspect --compact > "$RUNNER_TEMP/inspect.json" cmp evidence/git-dag-v1.json "$RUNNER_TEMP/inspect.json" + env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab pack-verify > "$RUNNER_TEMP/pack-verify.txt" + cmp docs/demo/git-pack-index-v1/verify.txt "$RUNNER_TEMP/pack-verify.txt" + env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab pack-inspect --compact > "$RUNNER_TEMP/pack-inspect.json" + cmp evidence/git-pack-index-v1.json "$RUNNER_TEMP/pack-inspect.json" - name: Check tests, evidence freshness, and shell boundary shell: bash @@ -71,9 +75,11 @@ jobs: set -euo pipefail env -u PYTHONHOME -u PYTHONPATH python -W error -m unittest discover -s tests -v env -u PYTHONHOME -u PYTHONPATH python -B tools/generate_evidence.py --check + env -u PYTHONHOME -u PYTHONPATH python -B tools/generate_pack_evidence.py --check bash -n tools/capture_report.sh + bash -n tools/capture_pack_report.sh test "$(shellcheck --version | awk '$1 == "version:" {print $2}')" = "0.9.0" - shellcheck tools/capture_report.sh + shellcheck tools/capture_report.sh tools/capture_pack_report.sh test "$(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none)" = "" distribution: @@ -119,6 +125,11 @@ jobs: env -u PYTHONHOME -u PYTHONPATH "$runtime/bin/python" -B -m git_dag_lab verify ) > "$RUNNER_TEMP/wheel-verify.txt" cmp docs/demo/git-dag-v1/verify.txt "$RUNNER_TEMP/wheel-verify.txt" + ( + cd "$RUNNER_TEMP" + env -u PYTHONHOME -u PYTHONPATH "$runtime/bin/python" -B -m git_dag_lab pack-verify + ) > "$RUNNER_TEMP/wheel-pack-verify.txt" + cmp docs/demo/git-pack-index-v1/verify.txt "$RUNNER_TEMP/wheel-pack-verify.txt" test "$(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none)" = "" - name: Upload verified distributions diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..430e0eb --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,46 @@ +name: CodeQL + +on: + push: + branches: + - main + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + security-events: write + +concurrency: + group: codeql-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + language: + - python + steps: + - name: Check out the immutable event revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@c4dd10e44af883a891fe31ced449bcb4a6728b9b # v3.37.6 + with: + languages: ${{ matrix.language }} + queries: security-extended + + - name: Analyze + uses: github/codeql-action/analyze@c4dd10e44af883a891fe31ced449bcb4a6728b9b # v3.37.6 + with: + category: /language:${{ matrix.language }} diff --git a/README.md b/README.md index 0273555..3c7c2a7 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # Git DAG Evidence Lab -> A dependency-free Python systems lab that constructs a real Git object database from plumbing commands and independently verifies every byte-addressed object. +[![CI](https://github.com/omar07ibrahim/hello-git/actions/workflows/ci.yml/badge.svg)](https://github.com/omar07ibrahim/hello-git/actions/workflows/ci.yml) [![CodeQL](https://github.com/omar07ibrahim/hello-git/actions/workflows/codeql.yml/badge.svg)](https://github.com/omar07ibrahim/hello-git/actions/workflows/codeql.yml) + +> A dependency-free Python systems lab that constructs real Git object storage, then independently verifies loose-object DAGs plus closed-subset pack v2/index v2 bytes. The experiment demonstrates a subtle but important property: a merge commit and a rebase-shaped replay can resolve to **exactly the same tree** while preserving **different histories**. ![Real offline Git DAG evidence report showing the verified graph, checks, CLI receipt, and object envelope](docs/assets/git-dag-report.png) -

Attested 1440×1800 Chromium capture of the checked-in offline report. No external assets, JavaScript, network, secrets, or host-repository data.

+

Attested 1440×1800 Chromium capture of the checked-in offline report. Provenance: report receipt `2da1ccd8…69c84`; screenshot SHA-256 `e539db11…e17e`. No external assets, JavaScript, network, secrets, or host-repository data.

## Quick start @@ -15,6 +17,8 @@ Requirements: Python 3.10+ and Git 2.29+. There are no runtime Python dependenci ```bash python3 -m git_dag_lab verify python3 -m git_dag_lab inspect +python3 -m git_dag_lab pack-verify +python3 -m git_dag_lab pack-inspect python3 -m unittest discover -s tests -v ``` @@ -34,6 +38,29 @@ The fixture contains three blobs, six trees, and five commits. `merge` stores th The replay is called **rebase-shaped** because it is constructed directly with `git commit-tree`. The lab does not claim to execute porcelain `git rebase`. + +## A second systems slice: verify pack and index bytes + +The pack path stores three fixed synthetic blobs in a fresh private SHA-1 repository, invokes real `git pack-objects`, then removes the repository after independently decoding both generated files. The CLI receipt below is actual stdout from that production path: + +![Exact real pack-verify stdout with actual pack and receipt digests](docs/assets/git-pack-cli.svg) + +The offline report is rendered from the same canonical receipt and captured by digest-pinned Chromium in a read-only, network-disabled container: + +![Actual pinned-Chromium screenshot of the verified pack v2 and index v2 report](docs/assets/git-pack-report.png) + +

Attested 1440×1500 Chromium capture. Provenance: pack report receipt `a884aed4…a81b0`; screenshot SHA-256 `ad1f615d…4d63`. Three deterministic synthetic blobs; no delta support, arbitrary-repository compatibility, authentication, network, or host data.

+ +The verifier does not trust the pack filename or Git's index. It parses the variable-length pack entry headers, bounds each zlib stream, reconstructs logical blob IDs, verifies the pack trailer, then parses the 256-entry cumulative fanout table, sorted OIDs, CRC32 rows, 32/64-bit offsets, pack binding, and index checksum. + +![Actual byte offsets and sizes decoded from the generated pack and index](docs/assets/git-pack-layout.svg) + +![Actual populated fanout buckets, sorted object IDs, and lookup ranges](docs/assets/git-pack-fanout.svg) + +![Receipt-derived integrity chain from pack header through index checksum](docs/assets/git-pack-integrity.svg) + +This is deliberately a closed subset: pack v2 and index v2, at most 64 objects, 1 MiB files, 256 KiB expanded objects, and non-delta entries only. Delta entries (OFS/REF), other object formats, arbitrary repositories, reachability, and pack optimization are not claimed. SHA-1 and CRC32 model Git storage integrity here; neither is presented as authentication, a signature, or collision-resistant security. + ## The hard part: verify Git without trusting Git Writing an object with Git and asking Git to identify it would only prove that Git agrees with itself. This lab reads the raw stored bytes and independently computes: @@ -51,7 +78,7 @@ SHA-1 is used because this scenario models a SHA-1 Git object database. Here it ## Isolation and execution boundaries - Git is resolved once to an absolute executable and invoked with argument arrays, never a shell. -- 10 local subcommands are allow-listed, while one fixed isolated `git init` creates the bare database; transport commands and remote-looking arguments are rejected. +- 11 local subcommands are allow-listed, including the bounded `pack-objects` path, while one fixed isolated `git init` creates each bare database; transport commands and remote-looking arguments are rejected. - `HOME`, `XDG_CONFIG_HOME`, and `TMPDIR` are private; inherited Git config, hooks, replacement objects, identity, and object-directory redirects are ignored. - fixed synthetic identity `dag-lab@example.invalid`, fixed UTC timestamps, and fixed LF payloads make object IDs reproducible. - symlinked workspace components are rejected; stdout/stderr are spooled privately and checked before bounded reads. @@ -61,24 +88,26 @@ See [SECURITY.md](SECURITY.md) for the threat model and trusted-input boundary. ## Evidence pipeline -Every README visual begins with the same canonical CLI document. The generator runs fresh experiments twice, requires byte-identical outputs, derives the SVG and offline HTML, and binds each artifact into a hash manifest. Digest-pinned Chromium then captures the report in a read-only container with `--network none`. A separate attestation binds the exact report, rendered DOM, PNG, browser binary/version, container digest, isolation policy, viewport, and capture-script hash; without that attestation, the generator refuses to call the screenshot verified. +Every README visual begins with a canonical production CLI document. The DAG and pack generators each run fresh experiments twice, require byte-identical outputs, derive their SVGs and offline HTML, and bind every artifact into a hash manifest. Digest-pinned Chromium captures both reports in read-only containers with `--network none`. Separate attestations bind each exact report, rendered DOM, PNG, browser binary/version, container digest, isolation policy, viewport, and capture-script hash; without the matching attestation, a generator refuses to call its screenshot verified. ![Architecture of the fixed scenario, real Git plumbing, independent verification, and evidence publication pipeline](docs/assets/evidence-pipeline.svg) ### Reproduce the checked-in evidence ```bash -# Verify JSON, transcripts, SVGs, HTML, source hashes, and the existing PNG. +# Verify both JSON/transcript/visual/report/manifest packages and PNG attestations. python3 -B tools/generate_evidence.py --check +python3 -B tools/generate_pack_evidence.py --check -# Rebuild evidence and recapture the report with the pinned browser container. +# Rebuild and recapture either offline report with pinned Chromium. tools/capture_report.sh +tools/capture_pack_report.sh -# Run all engine, boundary, CLI, evidence, and provenance tests. +# Run all parser, boundary, CLI, evidence, and provenance tests. python3 -W error -m unittest discover -s tests -v ``` -Current verified baseline: **61 tests**, **9/9 graph invariants**, **57 isolated Git invocations**, report receipt `2da1ccd8…69c84`, and screenshot SHA-256 `e539db11…e17e`. +Current verified baseline: **89 tests**, **9/9 graph invariants**, **7/7 pack/index checks**, **57 isolated Git invocations** in the DAG evidence run, two independently replayed evidence packages, and two attested offline browser captures. | Artifact | What it proves | |---|---| @@ -89,13 +118,22 @@ Current verified baseline: **61 tests**, **9/9 graph invariants**, **57 isolated | [`rendered-dom.html`](docs/demo/git-dag-v1/rendered-dom.html) | Actual DOM emitted by Chromium during the attested capture | | [`capture-attestation.json`](docs/demo/git-dag-v1/capture-attestation.json) | Report/DOM/PNG hashes plus verified browser, container, isolation, viewport, and script provenance | | [`manifest.json`](docs/demo/git-dag-v1/manifest.json) | SHA-256, byte size, role, source hashes, normalized argv, and attestation receipt | -| [`git-dag-report.png`](docs/assets/git-dag-report.png) | Actual Chromium rendering of the report at 1440×1800 | +| [`git-dag-report.png`](docs/assets/git-dag-report.png) | Actual Chromium rendering of the DAG report at 1440×1800 | +| [`evidence/git-pack-index-v1.json`](evidence/git-pack-index-v1.json) | Canonical real pack/index receipt, physical entry order, cross-bound rows, and non-claims | +| [`git-pack-cli.svg`](docs/assets/git-pack-cli.svg) | Exact production `pack-verify` stdout rendered as an accessible terminal panel | +| [`git-pack-layout.svg`](docs/assets/git-pack-layout.svg) | Actual pack offsets/sizes and index-table byte counts | +| [`git-pack-fanout.svg`](docs/assets/git-pack-fanout.svg) | Actual non-empty fanout buckets and sorted OID ranges | +| [`git-pack-integrity.svg`](docs/assets/git-pack-integrity.svg) | Receipt-derived pack/index checksum and row-binding workflow | +| [`git-pack-report.png`](docs/assets/git-pack-report.png) | Actual Chromium rendering of the pack/index report at 1440×1500 | +| [`git-pack-index-v1/manifest.json`](docs/demo/git-pack-index-v1/manifest.json) | Hash/size/source/command/capture inventory for every pack visual and output | ## Test coverage by risk The standard-library suite exercises more than happy-path graph construction: - independent blob, tree, and commit envelope hashes; +- pack v2 headers, bounded zlib streams, logical OIDs, trailer checksum, and explicit delta rejection; +- index v2 fanout, sorted OIDs, CRC32 rows, small/large offsets, pack binding, and checksum mutations; - exact object/ref inventories, parent ordering, reachability, and ancestry; - Git's special `directory/` tree ordering, truncated binary objects, and malformed headers; - hostile inherited Git environment and fake global identity/config; diff --git a/SECURITY.md b/SECURITY.md index e4cc6f4..37c677e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security model -Git DAG Evidence Lab creates a temporary bare repository below the selected workspace and removes it after each run. It does not inspect the repository that contains this source code. +Git DAG Evidence Lab creates a temporary bare repository below the selected workspace for each DAG or pack experiment and removes it after the run. It does not inspect the repository that contains this source code. The command boundary is deliberately narrow: @@ -13,6 +13,12 @@ The command boundary is deliberately narrow: - temporary roots with symlinked path components are rejected. - stdout and stderr are captured in private temporary files and rejected before loading into memory when either exceeds 1 MiB. +## Pack/index closed subset + +The pack experiment passes only three fixed synthetic blob IDs to `git pack-objects`; it does not accept a repository path, revision, ref, or caller-provided object list. Generated `.pack` and `.idx` files must be regular, single-link files no larger than 1 MiB and must remain the same inode and size across the bounded read. + +The independent parser accepts pack v2 and index v2 only. It rejects OFS/REF deltas, more than 64 objects, objects expanding beyond 256 KiB, invalid or unterminated zlib streams, duplicate logical objects, non-canonical fanout/large-offset tables, and any mismatch among logical object IDs, CRC32 rows, offsets, pack trailer, index pack binding, or index checksum. These checks establish the fixed fixture's storage integrity; they do not establish provenance, authenticity, repository reachability, or safety of arbitrary Git data. + ## SHA-1 scope The lab uses SHA-1 because the scenario explicitly models a SHA-1 Git object database. The independent envelope calculation demonstrates deterministic content addressing and detects accidental changes in these fixtures. It is not a signature, authentication mechanism, or claim of modern collision resistance. diff --git a/docs/assets/git-pack-cli.svg b/docs/assets/git-pack-cli.svg new file mode 100644 index 0000000..31d1063 --- /dev/null +++ b/docs/assets/git-pack-cli.svg @@ -0,0 +1,29 @@ + + Real Git pack/index CLI receipt + Exact stdout from pack-verify; the command executed real git pack-objects and independently checked both files. + {"report_receipt_sha256":"a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0","source":"evidence/git-pack-index-v1.json"} + + + + + + + + REAL CLI STDOUT · STDERR 0 BYTES · EXIT 0 + + + $ python3 -B -m git_dag_lab pack-verify + PASS git-pack-index-lab/v1 objects=3 pack_version=2 index_version=2 deltas=0 + pack_sha1=146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5 + receipt_sha256=a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0 + verify.txt SHA-256 · ccab273b6f07dc72a28ad3e1065545e6bd535f766d96ad6c3dad273eea432f31 + Report receipt SHA-256 · a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0 + diff --git a/docs/assets/git-pack-fanout.svg b/docs/assets/git-pack-fanout.svg new file mode 100644 index 0000000..9c76cf8 --- /dev/null +++ b/docs/assets/git-pack-fanout.svg @@ -0,0 +1,45 @@ + + Actual index v2 fanout lookup + The three populated prefix buckets, sorted object IDs, and exact index ranges come from the verified index receipt. + {"report_receipt_sha256":"a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0","source":"evidence/git-pack-index-v1.json"} + + + + + + + + 256-BUCKET CUMULATIVE FANOUT · THREE ACTUAL OBJECT IDS + Prefix lookup narrows the sorted OID table. + + PREFIX ac + [0, 1) + + + content-addressing · blob · 26 bytes + ac0913a01c3a10dbcdf22143ad8f53d015774ff7 + + PREFIX fd + [1, 2) + + + index-fanout · blob · 60 bytes + fda2e7ba229f29298234d2568c24b04c04e603df + + PREFIX fe + [2, 3) + + + binary-header · blob · 32 bytes + fefa1cc823e0ec463b7923972b03901b52808fc1 + + Every displayed bucket and range is decoded from index bytes · receipt a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0 + diff --git a/docs/assets/git-pack-integrity.svg b/docs/assets/git-pack-integrity.svg new file mode 100644 index 0000000..e51e47b --- /dev/null +++ b/docs/assets/git-pack-integrity.svg @@ -0,0 +1,54 @@ + + Independent pack/index integrity chain + The production verifier reconstructs each logical object, checks entry CRCs and offsets, then binds both file checksums. + {"report_receipt_sha256":"a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0","source":"evidence/git-pack-index-v1.json"} + + + + + + + + GIT WRITES · PYTHON INDEPENDENTLY READS · NO MODELLED SUCCESS + Five checks connect raw bytes to one receipt. + + 01 + PACK HEADER + v2 · 3 objects + PASS + + + 02 + ENTRY STREAMS + bounded zlib · logic… + PASS + + + 03 + INDEX ROWS + CRC32 · exact offsets + PASS + + + 04 + PACK BINDING + 146ea7519b9beee2ef83… + PASS + + + 05 + INDEX CHECKSUM + 3cb00d5dd06610c624be… + PASS + + 3/3 CRC32 and offsets cross-match · 0 deltas accepted · pack SHA-1 146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5 + SHA-1 models Git object storage; authentication claim = false · receipt a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0 + diff --git a/docs/assets/git-pack-layout.svg b/docs/assets/git-pack-layout.svg new file mode 100644 index 0000000..df605e5 --- /dev/null +++ b/docs/assets/git-pack-layout.svg @@ -0,0 +1,63 @@ + + Actual Git pack v2 and index v2 byte layout + Byte offsets and section sizes are derived from the real production pack receipt. + {"index_sha256":"c4f82d0cb1685f3d3b268575152993408c0ee4f00de346a3482b06c6fe59eebc","pack_sha256":"d8c8d5db126731f265cdcfe17212022001a117d6e7c2300b7939a25b93cb4052","report_receipt_sha256":"a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0","source":"evidence/git-pack-index-v1.json"} + + + + + + + + REAL PACK V2 · ACTUAL OFFSETS · THREE NON-DELTA BLOBS + 189 pack bytes, independently decoded + + + PACK + + 01 · content-addressing + offset 12 + 39 bytes + + 02 · index-fanout + offset 51 + 73 bytes + + 03 · binary-header + offset 124 + 45 bytes + + SHA-1 + 20 B + INDEX V2 FIXED TABLES + + header + 8 B + + fanout[256] + 1024 B + + sorted OIDs + 60 B + + CRC32 + 12 B + + offsets + 12 B + + checksums + 40 B + + Pack trailer SHA-1 · 146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5 + Index SHA-1 · 3cb00d5dd06610c624befb4e9e1b40f9cdbf9dfa · binds the same pack checksum + Receipt SHA-256 · a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0 + diff --git a/docs/assets/git-pack-report.png b/docs/assets/git-pack-report.png new file mode 100644 index 0000000..cd58cae Binary files /dev/null and b/docs/assets/git-pack-report.png differ diff --git a/docs/demo/git-dag-v1/manifest.json b/docs/demo/git-dag-v1/manifest.json index a57a689..f9504a6 100644 --- a/docs/demo/git-dag-v1/manifest.json +++ b/docs/demo/git-dag-v1/manifest.json @@ -130,13 +130,13 @@ "sources": [ { "path": "git_dag_lab/lab.py", - "sha256": "8c4bf11d5c7840a7dcbe3f4486246aff9b3a306c6bceb7b15be3fd0040a7bbd4", - "size": 34652 + "sha256": "289e688e84024c20b94f3aa2e59be693b81b507b8581fe5547cc8f5eb223879b", + "size": 34676 }, { "path": "git_dag_lab/cli.py", - "sha256": "4b8b2433e0b613a303b886d08a630ddfeada557702fc47de4cd2f59824e6501d", - "size": 1684 + "sha256": "03648331670cdc350fd486b2939e4914e90ee7ca7c2ea2b5cabd4eced21fe766", + "size": 2335 }, { "path": "tools/generate_evidence.py", diff --git a/docs/demo/git-pack-index-v1/capture-attestation.json b/docs/demo/git-pack-index-v1/capture-attestation.json new file mode 100644 index 0000000..cbbb156 --- /dev/null +++ b/docs/demo/git-pack-index-v1/capture-attestation.json @@ -0,0 +1,60 @@ +{ + "attestation": { + "browser": { + "binary_path": "/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell", + "sha256": "003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d", + "version": "Chromium 140.0.7339.186" + }, + "container": { + "architecture": "amd64", + "image": "mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c" + }, + "input": { + "report": { + "path": "docs/demo/git-pack-index-v1/report.html", + "report_receipt_sha256": "a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0", + "sha256": "0109d54e1e3f794fe3422ddf7720fd8c322c3ee3d82dc41c8d3ff1207f250624", + "size": 5268 + } + }, + "isolation": { + "capabilities": "all-dropped", + "demo_mount": "read-only", + "network": "none", + "no_new_privileges": true, + "pull": "never", + "root_filesystem": "read-only", + "user": "caller-nonroot" + }, + "outputs": { + "rendered_dom": { + "path": "docs/demo/git-pack-index-v1/rendered-dom.html", + "sha256": "736bd86208630f5e1437335e009615bffd8f7bb96932587acad33b3307f1827d", + "size": 5268 + }, + "screenshot": { + "height": 1500, + "path": "docs/assets/git-pack-report.png", + "sha256": "ad1f615da03d2e0570a4b3b26603c310058964bfbc5dda201d176de8107e4d63", + "size": 590359, + "width": 1440 + } + }, + "schema_version": "git-pack-browser-capture-attestation/v1", + "script": { + "path": "tools/capture_pack_report.sh", + "sha256": "3477eeec4a10026fe95b87803f68657069c77f2b491bd69d3cddd56777924f0b", + "size": 11083 + }, + "viewport": { + "device_scale_factor": 1, + "height": 1500, + "width": 1440 + } + }, + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": "9709eafc99c848f825d083b795a9ba2802109d76b21289e2a4f72c445519c35b" + } +} diff --git a/docs/demo/git-pack-index-v1/inspect.json b/docs/demo/git-pack-index-v1/inspect.json new file mode 100644 index 0000000..37679af --- /dev/null +++ b/docs/demo/git-pack-index-v1/inspect.json @@ -0,0 +1,127 @@ +{ + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": "a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0" + }, + "report": { + "checks": { + "all_fixture_objects_present": true, + "delta_entries_absent": true, + "index_checksum_verified": true, + "index_crc32_matches_pack": true, + "index_fanout_matches_sorted_oids": true, + "index_offsets_match_pack": true, + "pack_trailer_verified": true + }, + "command_trace": [ + "init", + "hash-object", + "hash-object", + "hash-object", + "pack-objects" + ], + "fixture": { + "object_count": 3, + "objects": [ + { + "label": "binary-header", + "oid": "fefa1cc823e0ec463b7923972b03901b52808fc1", + "payload_sha256": "630dcd2966c4336691125448bbb25b4ff412a49c732db2c8abc1b8581bd710dd", + "size": 32 + }, + { + "label": "content-addressing", + "oid": "ac0913a01c3a10dbcdf22143ad8f53d015774ff7", + "payload_sha256": "218787ebc53eb8b381705f18f614758dc0b2f7e2781da3eb8132a0eda7aef33d", + "size": 26 + }, + { + "label": "index-fanout", + "oid": "fda2e7ba229f29298234d2568c24b04c04e603df", + "payload_sha256": "5d91ab3e2e99923d07e6f7c38354d19f84f2d1b2bbcc0faa8ceb14a71f9868cc", + "size": 60 + } + ] + }, + "index": { + "bytes": 1156, + "index_sha1": "3cb00d5dd06610c624befb4e9e1b40f9cdbf9dfa", + "nonzero_fanout_buckets": [ + { + "cumulative": 1, + "prefix": "ac", + "range_start": 0 + }, + { + "cumulative": 2, + "prefix": "fd", + "range_start": 1 + }, + { + "cumulative": 3, + "prefix": "fe", + "range_start": 2 + } + ], + "pack_sha1": "146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5", + "sha256": "c4f82d0cb1685f3d3b268575152993408c0ee4f00de346a3482b06c6fe59eebc", + "version": 2 + }, + "object_format": "sha1", + "objects_in_pack_order": [ + { + "crc32": "fb8c64c5", + "index_crc32": "fb8c64c5", + "index_offset": 12, + "label": "content-addressing", + "object_type": "blob", + "offset": 12, + "oid": "ac0913a01c3a10dbcdf22143ad8f53d015774ff7", + "packed_size": 39, + "payload_sha256": "218787ebc53eb8b381705f18f614758dc0b2f7e2781da3eb8132a0eda7aef33d", + "size": 26 + }, + { + "crc32": "c532220c", + "index_crc32": "c532220c", + "index_offset": 51, + "label": "index-fanout", + "object_type": "blob", + "offset": 51, + "oid": "fda2e7ba229f29298234d2568c24b04c04e603df", + "packed_size": 73, + "payload_sha256": "5d91ab3e2e99923d07e6f7c38354d19f84f2d1b2bbcc0faa8ceb14a71f9868cc", + "size": 60 + }, + { + "crc32": "88dbb9d0", + "index_crc32": "88dbb9d0", + "index_offset": 124, + "label": "binary-header", + "object_type": "blob", + "offset": 124, + "oid": "fefa1cc823e0ec463b7923972b03901b52808fc1", + "packed_size": 45, + "payload_sha256": "630dcd2966c4336691125448bbb25b4ff412a49c732db2c8abc1b8581bd710dd", + "size": 32 + } + ], + "pack": { + "bytes": 189, + "delta_count": 0, + "object_count": 3, + "sha256": "d8c8d5db126731f265cdcfe17212022001a117d6e7c2300b7939a25b93cb4052", + "trailer_sha1": "146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5", + "version": 2 + }, + "schema_version": "git-pack-index-lab/v1", + "scope": { + "authentication_claim": false, + "delta_entries_supported": false, + "fixture_kind": "three deterministic synthetic blobs", + "git_pack_objects_executed": true, + "network_required": false + } + } +} diff --git a/docs/demo/git-pack-index-v1/manifest.json b/docs/demo/git-pack-index-v1/manifest.json new file mode 100644 index 0000000..4ecdcdc --- /dev/null +++ b/docs/demo/git-pack-index-v1/manifest.json @@ -0,0 +1,152 @@ +{ + "artifacts": [ + { + "path": "docs/assets/git-pack-cli.svg", + "role": "visualized exact pack-verify transcript", + "sha256": "ccd4e4955bcba485bb4bd34d493bde723920dd24b95cbfd465a8373165a38727", + "size": 2796 + }, + { + "path": "docs/assets/git-pack-fanout.svg", + "role": "actual index fanout ranges and sorted object IDs", + "sha256": "bfb0e720f9e82083803dbebcac46549f2f4725122364efa173cd5007342c0a4f", + "size": 3803 + }, + { + "path": "docs/assets/git-pack-integrity.svg", + "role": "pack/index integrity chain derived from the receipt", + "sha256": "6733240058cdfbd114163798d7e48df287edb6c380380a40c14f907772f01ee3", + "size": 4308 + }, + { + "path": "docs/assets/git-pack-layout.svg", + "role": "actual pack and index byte layout", + "sha256": "77703a115b270f49b00d573ede12a9537efe9c33fcadacc21ee1e82284db11cd", + "size": 5721 + }, + { + "path": "docs/assets/git-pack-report.png", + "role": "attested pack/index offline report browser capture", + "sha256": "ad1f615da03d2e0570a4b3b26603c310058964bfbc5dda201d176de8107e4d63", + "size": 590359 + }, + { + "path": "docs/demo/git-pack-index-v1/capture-attestation.json", + "role": "pack capture provenance and isolation attestation", + "sha256": "18e5fea29af65a0fd15b2959486b282f75425f1dabb1df20a556b57bf3c22757", + "size": 2011 + }, + { + "path": "docs/demo/git-pack-index-v1/inspect.json", + "role": "exact pretty pack-inspect stdout", + "sha256": "527650dfc891bb808688ea1cf53198db5aa6321866a3e302f5c70de6193bf921", + "size": 3851 + }, + { + "path": "docs/demo/git-pack-index-v1/rendered-dom.html", + "role": "actual DOM emitted during the pack report capture", + "sha256": "736bd86208630f5e1437335e009615bffd8f7bb96932587acad33b3307f1827d", + "size": 5268 + }, + { + "path": "docs/demo/git-pack-index-v1/report.html", + "role": "dependency-free pack/index offline report", + "sha256": "0109d54e1e3f794fe3422ddf7720fd8c322c3ee3d82dc41c8d3ff1207f250624", + "size": 5268 + }, + { + "path": "docs/demo/git-pack-index-v1/verify.txt", + "role": "exact pack-verify stdout", + "sha256": "ccab273b6f07dc72a28ad3e1065545e6bd535f766d96ad6c3dad273eea432f31", + "size": 208 + }, + { + "path": "evidence/git-pack-index-v1.json", + "role": "canonical compact pack/index CLI evidence", + "sha256": "a33a5a3a3a4930defec6309013415a18de0527b18a3adcb6e2c5aebce06b4b13", + "size": 2770 + } + ], + "capture": { + "attestation": "docs/demo/git-pack-index-v1/capture-attestation.json", + "attestation_receipt_sha256": "9709eafc99c848f825d083b795a9ba2802109d76b21289e2a4f72c445519c35b", + "browser_binary": "/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell", + "browser_sha256": "003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d", + "browser_version": "Chromium 140.0.7339.186", + "container_image": "mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c", + "network": "none", + "rendered_dom_sha256": "736bd86208630f5e1437335e009615bffd8f7bb96932587acad33b3307f1827d", + "screenshot_sha256": "ad1f615da03d2e0570a4b3b26603c310058964bfbc5dda201d176de8107e4d63", + "status": "attested", + "viewport": { + "height": 1500, + "width": 1440 + } + }, + "commands": [ + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-verify" + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": "docs/demo/git-pack-index-v1/verify.txt" + }, + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-inspect", + "--compact" + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": "evidence/git-pack-index-v1.json" + }, + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-inspect" + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": "docs/demo/git-pack-index-v1/inspect.json" + } + ], + "report_receipt_sha256": "a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0", + "schema_version": "git-pack-index-evidence-manifest/v1", + "sources": [ + { + "path": "git_dag_lab/pack.py", + "sha256": "d3882af57ad9b3d85580032b7e5ceccd2c8b443fbee7be30c7452a31aee3f8a8", + "size": 20802 + }, + { + "path": "git_dag_lab/cli.py", + "sha256": "03648331670cdc350fd486b2939e4914e90ee7ca7c2ea2b5cabd4eced21fe766", + "size": 2335 + }, + { + "path": "tools/generate_pack_evidence.py", + "sha256": "2a85d4e5d3a98cca6e562eae7fad28e5f64d61bd02f973dcffdade274b89514d", + "size": 41315 + }, + { + "path": "tools/capture_pack_report.sh", + "sha256": "3477eeec4a10026fe95b87803f68657069c77f2b491bd69d3cddd56777924f0b", + "size": 11083 + } + ] +} diff --git a/docs/demo/git-pack-index-v1/rendered-dom.html b/docs/demo/git-pack-index-v1/rendered-dom.html new file mode 100644 index 0000000..03a6b84 --- /dev/null +++ b/docs/demo/git-pack-index-v1/rendered-dom.html @@ -0,0 +1,33 @@ + + + + + + Git pack/index evidence — verified offline report + + + +
+

Git pack/index lab · actual offline report

+

Pack v2.
Index v2.

+

Real git pack-objects output is decoded independently: bounded zlib entries, logical object IDs, fanout ranges, CRC32 rows, offsets, and both file checksums.

+
PASS · report receipt a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0
+
+
3packed objects
+
189actual pack bytes
+
1156actual index bytes
+
7/7checks passed
+
+
+

Objects in physical pack order

fixtureOIDsizeoffsetCRC32
content-addressingac0913a01c3a10dbcdf22143ad8f53d015774ff72612fb8c64c5
index-fanoutfda2e7ba229f29298234d2568c24b04c04e603df6051c532220c
binary-headerfefa1cc823e0ec463b7923972b03901b52808fc13212488dbb9d0
+

Independent gate

  • ✓all fixture objects present
  • ✓delta entries absent
  • ✓index checksum verified
  • ✓index crc32 matches pack
  • ✓index fanout matches sorted oids
  • ✓index offsets match pack
  • ✓pack trailer verified
+
+

Real CLI receipt

$ python3 -B -m git_dag_lab pack-verify
+PASS git-pack-index-lab/v1 objects=3 pack_version=2 index_version=2 deltas=0 pack_sha1=146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5 receipt_sha256=a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0
Pack trailer
146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5

Index checksum
3cb00d5dd06610c624befb4e9e1b40f9cdbf9dfa

Delta support: false · authentication claim: false · network required: false

+ +
+ + + diff --git a/docs/demo/git-pack-index-v1/report.html b/docs/demo/git-pack-index-v1/report.html new file mode 100644 index 0000000..506dc64 --- /dev/null +++ b/docs/demo/git-pack-index-v1/report.html @@ -0,0 +1,33 @@ + + + + + + + Git pack/index evidence — verified offline report + + + +
+

Git pack/index lab · actual offline report

+

Pack v2.
Index v2.

+

Real git pack-objects output is decoded independently: bounded zlib entries, logical object IDs, fanout ranges, CRC32 rows, offsets, and both file checksums.

+
PASS · report receipt a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0
+
+
3packed objects
+
189actual pack bytes
+
1156actual index bytes
+
7/7checks passed
+
+
+

Objects in physical pack order

fixtureOIDsizeoffsetCRC32
content-addressingac0913a01c3a10dbcdf22143ad8f53d015774ff72612fb8c64c5
index-fanoutfda2e7ba229f29298234d2568c24b04c04e603df6051c532220c
binary-headerfefa1cc823e0ec463b7923972b03901b52808fc13212488dbb9d0
+

Independent gate

  • ✓all fixture objects present
  • ✓delta entries absent
  • ✓index checksum verified
  • ✓index crc32 matches pack
  • ✓index fanout matches sorted oids
  • ✓index offsets match pack
  • ✓pack trailer verified
+
+

Real CLI receipt

$ python3 -B -m git_dag_lab pack-verify
+PASS git-pack-index-lab/v1 objects=3 pack_version=2 index_version=2 deltas=0 pack_sha1=146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5 receipt_sha256=a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0
Pack trailer
146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5

Index checksum
3cb00d5dd06610c624befb4e9e1b40f9cdbf9dfa

Delta support: false · authentication claim: false · network required: false

+ +
+ + diff --git a/docs/demo/git-pack-index-v1/verify.txt b/docs/demo/git-pack-index-v1/verify.txt new file mode 100644 index 0000000..652c468 --- /dev/null +++ b/docs/demo/git-pack-index-v1/verify.txt @@ -0,0 +1 @@ +PASS git-pack-index-lab/v1 objects=3 pack_version=2 index_version=2 deltas=0 pack_sha1=146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5 receipt_sha256=a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0 diff --git a/evidence/git-pack-index-v1.json b/evidence/git-pack-index-v1.json new file mode 100644 index 0000000..eb8219e --- /dev/null +++ b/evidence/git-pack-index-v1.json @@ -0,0 +1 @@ +{"receipt":{"algorithm":"sha256","canonicalization":"UTF-8 JSON; sorted keys; compact separators","sha256":"a884aed43dbb027559f6df78ff727863b9b52b145ffb719756b2302a490a81b0"},"report":{"checks":{"all_fixture_objects_present":true,"delta_entries_absent":true,"index_checksum_verified":true,"index_crc32_matches_pack":true,"index_fanout_matches_sorted_oids":true,"index_offsets_match_pack":true,"pack_trailer_verified":true},"command_trace":["init","hash-object","hash-object","hash-object","pack-objects"],"fixture":{"object_count":3,"objects":[{"label":"binary-header","oid":"fefa1cc823e0ec463b7923972b03901b52808fc1","payload_sha256":"630dcd2966c4336691125448bbb25b4ff412a49c732db2c8abc1b8581bd710dd","size":32},{"label":"content-addressing","oid":"ac0913a01c3a10dbcdf22143ad8f53d015774ff7","payload_sha256":"218787ebc53eb8b381705f18f614758dc0b2f7e2781da3eb8132a0eda7aef33d","size":26},{"label":"index-fanout","oid":"fda2e7ba229f29298234d2568c24b04c04e603df","payload_sha256":"5d91ab3e2e99923d07e6f7c38354d19f84f2d1b2bbcc0faa8ceb14a71f9868cc","size":60}]},"index":{"bytes":1156,"index_sha1":"3cb00d5dd06610c624befb4e9e1b40f9cdbf9dfa","nonzero_fanout_buckets":[{"cumulative":1,"prefix":"ac","range_start":0},{"cumulative":2,"prefix":"fd","range_start":1},{"cumulative":3,"prefix":"fe","range_start":2}],"pack_sha1":"146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5","sha256":"c4f82d0cb1685f3d3b268575152993408c0ee4f00de346a3482b06c6fe59eebc","version":2},"object_format":"sha1","objects_in_pack_order":[{"crc32":"fb8c64c5","index_crc32":"fb8c64c5","index_offset":12,"label":"content-addressing","object_type":"blob","offset":12,"oid":"ac0913a01c3a10dbcdf22143ad8f53d015774ff7","packed_size":39,"payload_sha256":"218787ebc53eb8b381705f18f614758dc0b2f7e2781da3eb8132a0eda7aef33d","size":26},{"crc32":"c532220c","index_crc32":"c532220c","index_offset":51,"label":"index-fanout","object_type":"blob","offset":51,"oid":"fda2e7ba229f29298234d2568c24b04c04e603df","packed_size":73,"payload_sha256":"5d91ab3e2e99923d07e6f7c38354d19f84f2d1b2bbcc0faa8ceb14a71f9868cc","size":60},{"crc32":"88dbb9d0","index_crc32":"88dbb9d0","index_offset":124,"label":"binary-header","object_type":"blob","offset":124,"oid":"fefa1cc823e0ec463b7923972b03901b52808fc1","packed_size":45,"payload_sha256":"630dcd2966c4336691125448bbb25b4ff412a49c732db2c8abc1b8581bd710dd","size":32}],"pack":{"bytes":189,"delta_count":0,"object_count":3,"sha256":"d8c8d5db126731f265cdcfe17212022001a117d6e7c2300b7939a25b93cb4052","trailer_sha1":"146ea7519b9beee2ef83e6c0d5cb80c8cf0593b5","version":2},"schema_version":"git-pack-index-lab/v1","scope":{"authentication_claim":false,"delta_entries_supported":false,"fixture_kind":"three deterministic synthetic blobs","git_pack_objects_executed":true,"network_required":false}}} diff --git a/git_dag_lab/__init__.py b/git_dag_lab/__init__.py index 4d5c901..35ff0d1 100644 --- a/git_dag_lab/__init__.py +++ b/git_dag_lab/__init__.py @@ -1,14 +1,19 @@ """Deterministic experiments with real Git objects.""" from .lab import LabError, LabReport, git_object_oid, parse_commit, parse_tree, run_lab +from .pack import PackReport, parse_index, parse_pack, run_pack_lab __all__ = [ "LabError", "LabReport", + "PackReport", "git_object_oid", + "parse_index", + "parse_pack", "parse_commit", "parse_tree", "run_lab", + "run_pack_lab", ] __version__ = "0.1.0" diff --git a/git_dag_lab/cli.py b/git_dag_lab/cli.py index 4528062..95185c3 100644 --- a/git_dag_lab/cli.py +++ b/git_dag_lab/cli.py @@ -9,6 +9,7 @@ from typing import TextIO from .lab import LabError, run_lab +from .pack import run_pack_lab def build_parser() -> argparse.ArgumentParser: @@ -27,6 +28,20 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="emit canonical JSON on one line instead of indented JSON", ) + + subparsers.add_parser( + "pack-verify", + help="build and independently verify a real Git pack v2/index v2 pair", + ) + pack_inspect_parser = subparsers.add_parser( + "pack-inspect", + help="print the complete machine-readable pack/index evidence document", + ) + pack_inspect_parser.add_argument( + "--compact", + action="store_true", + help="emit canonical JSON on one line instead of indented JSON", + ) return parser @@ -44,12 +59,15 @@ def main( args = build_parser().parse_args(argv) try: - report = run_lab(root if root is not None else Path.cwd()) + if args.command.startswith("pack-"): + report = run_pack_lab(root if root is not None else Path.cwd()) + else: + report = run_lab(root if root is not None else Path.cwd()) except LabError as exc: errors.write(f"ERROR git-dag-lab: {exc}\n") return 1 - if args.command == "verify": + if args.command in {"verify", "pack-verify"}: output.write(report.receipt_line + "\n") else: output.write(report.to_json(pretty=not args.compact) + "\n") diff --git a/git_dag_lab/lab.py b/git_dag_lab/lab.py index 21a79b5..3e70cd6 100644 --- a/git_dag_lab/lab.py +++ b/git_dag_lab/lab.py @@ -54,6 +54,7 @@ "hash-object", "merge-base", "mktree", + "pack-objects", "rev-list", "symbolic-ref", "update-ref", diff --git a/git_dag_lab/pack.py b/git_dag_lab/pack.py new file mode 100644 index 0000000..606b50b --- /dev/null +++ b/git_dag_lab/pack.py @@ -0,0 +1,613 @@ +"""Build and independently verify a bounded, deterministic Git pack/index pair. + +The fixture uses real ``git pack-objects`` output but validates the pack v2 and +index v2 bytes with Python's standard library. Delta entries are disabled for +this first closed subset and rejected by the parser. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import binascii +import hashlib +import json +import os +from pathlib import Path +import stat +import tempfile +from typing import Any +import zlib + +from .lab import ( + LabError, + VerificationError, + _GitRunner, + _canonical_json, + _deep_freeze, + _find_git, + _mutable_copy, + _new_workspace, + _store_blob, + _validate_root, + git_object_oid, +) + +PACK_SCHEMA_VERSION = "git-pack-index-lab/v1" +PACK_VERSION = 2 +INDEX_VERSION = 2 +PACK_MAGIC = b"PACK" +INDEX_MAGIC = b"\xfftOc" +MAX_PACK_BYTES = 1_048_576 +MAX_PACK_OBJECTS = 64 +MAX_OBJECT_BYTES = 262_144 + +PACK_BLOBS = ( + ("binary-header", bytes(range(32))), + ("content-addressing", b"content-addressed systems\n"), + ( + "index-fanout", + b"fanout tables map object-id prefixes to sorted index ranges\n", + ), +) +_TYPE_BY_CODE = {1: "commit", 2: "tree", 3: "blob", 4: "tag"} + + +@dataclass(frozen=True, slots=True) +class PackEntry: + """One independently decoded non-delta pack entry.""" + + crc32: int + object_type: str + offset: int + oid: str + packed_size: int + payload_sha256: str + size: int + + def as_dict(self) -> dict[str, object]: + return { + "crc32": f"{self.crc32:08x}", + "object_type": self.object_type, + "offset": self.offset, + "oid": self.oid, + "packed_size": self.packed_size, + "payload_sha256": self.payload_sha256, + "size": self.size, + } + + +@dataclass(frozen=True, slots=True) +class ParsedPack: + """Verified pack header, entries, and trailer.""" + + entries: tuple[PackEntry, ...] + trailer_sha1: str + version: int + + +@dataclass(frozen=True, slots=True) +class IndexEntry: + """One verified index v2 row.""" + + crc32: int + offset: int + oid: str + + def as_dict(self) -> dict[str, object]: + return { + "crc32": f"{self.crc32:08x}", + "offset": self.offset, + "oid": self.oid, + } + + +@dataclass(frozen=True, slots=True) +class ParsedIndex: + """Verified index v2 fanout, rows, and checksums.""" + + entries: tuple[IndexEntry, ...] + fanout: tuple[int, ...] + index_sha1: str + pack_sha1: str + version: int + + +@dataclass(frozen=True, slots=True) +class PackReport: + """Canonical evidence document for one real pack/index build.""" + + payload: Mapping[str, Any] + receipt_sha256: str + + @property + def document(self) -> dict[str, Any]: + return { + "report": _mutable_copy(self.payload), + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": self.receipt_sha256, + }, + } + + @property + def receipt_line(self) -> str: + pack = self.payload["pack"] + index = self.payload["index"] + return ( + f"PASS {PACK_SCHEMA_VERSION} " + f"objects={pack['object_count']} " + f"pack_version={pack['version']} " + f"index_version={index['version']} " + "deltas=0 " + f"pack_sha1={pack['trailer_sha1']} " + f"receipt_sha256={self.receipt_sha256}" + ) + + def to_json(self, *, pretty: bool = False) -> str: + if pretty: + return json.dumps( + self.document, + ensure_ascii=True, + indent=2, + sort_keys=True, + ) + return _canonical_json(self.document).decode("utf-8") + + +def _uint32(content: bytes, offset: int, *, label: str) -> int: + end = offset + 4 + if offset < 0 or end > len(content): + raise VerificationError(f"{label} is truncated") + return int.from_bytes(content[offset:end], "big") + + +def _uint64(content: bytes, offset: int, *, label: str) -> int: + end = offset + 8 + if offset < 0 or end > len(content): + raise VerificationError(f"{label} is truncated") + return int.from_bytes(content[offset:end], "big") + + +def _read_regular_file(path: Path, *, label: str) -> bytes: + """Read one bounded, single-link regular file without following a symlink.""" + + try: + before = path.lstat() + except OSError as exc: + raise VerificationError(f"{label} is unavailable") from exc + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or before.st_size < 1 + or before.st_size > MAX_PACK_BYTES + ): + raise VerificationError(f"{label} is not a bounded single-link file") + + flags = os.O_RDONLY + flags |= getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise VerificationError(f"{label} could not be opened safely") from exc + try: + observed = os.fstat(descriptor) + if ( + not stat.S_ISREG(observed.st_mode) + or observed.st_nlink != 1 + or observed.st_dev != before.st_dev + or observed.st_ino != before.st_ino + or observed.st_size != before.st_size + ): + raise VerificationError(f"{label} changed before it was read") + chunks: list[bytes] = [] + remaining = observed.st_size + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + raise VerificationError(f"{label} ended before its declared size") + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise VerificationError(f"{label} grew while it was read") + finally: + os.close(descriptor) + return b"".join(chunks) + + +def _decode_entry_header( + content: bytes, + offset: int, + *, + end: int, +) -> tuple[str, int, int]: + if offset >= end: + raise VerificationError("pack entry header is truncated") + first = content[offset] + offset += 1 + type_code = (first >> 4) & 0x07 + if type_code not in _TYPE_BY_CODE: + if type_code in {6, 7}: + raise VerificationError("delta pack entries are outside the reviewed subset") + raise VerificationError("pack entry type is invalid") + + size = first & 0x0F + shift = 4 + current = first + while current & 0x80: + if offset >= end or shift > 60: + raise VerificationError("pack entry size header is malformed") + current = content[offset] + offset += 1 + size |= (current & 0x7F) << shift + shift += 7 + if size > MAX_OBJECT_BYTES: + raise VerificationError("pack entry expands beyond the reviewed bound") + return _TYPE_BY_CODE[type_code], size, offset + + +def parse_pack(content: bytes) -> ParsedPack: + """Parse and verify one bounded SHA-1 pack v2 without invoking Git.""" + + if ( + type(content) is not bytes + or len(content) < 32 + or len(content) > MAX_PACK_BYTES + ): + raise VerificationError("pack bytes are outside the reviewed bound") + if content[:4] != PACK_MAGIC: + raise VerificationError("pack signature is invalid") + version = _uint32(content, 4, label="pack version") + if version != PACK_VERSION: + raise VerificationError("pack version is outside the reviewed subset") + count = _uint32(content, 8, label="pack object count") + if count < 1 or count > MAX_PACK_OBJECTS: + raise VerificationError("pack object count is outside the reviewed bound") + + pack_end = len(content) - 20 + trailer = content[pack_end:] + if hashlib.sha1(content[:pack_end], usedforsecurity=False).digest() != trailer: + raise VerificationError("pack trailer does not match the preceding bytes") + + entries: list[PackEntry] = [] + offset = 12 + seen: set[str] = set() + for _ in range(count): + entry_start = offset + object_type, declared_size, payload_start = _decode_entry_header( + content, + offset, + end=pack_end, + ) + inflater = zlib.decompressobj() + try: + payload = inflater.decompress( + content[payload_start:pack_end], + MAX_OBJECT_BYTES + 1, + ) + except zlib.error as exc: + raise VerificationError("pack entry zlib stream is invalid") from exc + if len(payload) > MAX_OBJECT_BYTES: + raise VerificationError("pack entry expands beyond the reviewed bound") + if not inflater.eof or inflater.unconsumed_tail: + raise VerificationError("pack entry zlib stream is incomplete or oversized") + consumed = pack_end - payload_start - len(inflater.unused_data) + if consumed < 1: + raise VerificationError("pack entry has an empty zlib stream") + offset = payload_start + consumed + if offset > pack_end or len(payload) != declared_size: + raise VerificationError("pack entry size does not match its payload") + + oid = git_object_oid(object_type, payload) + if oid in seen: + raise VerificationError("pack contains a duplicate logical object") + seen.add(oid) + packed = content[entry_start:offset] + entries.append( + PackEntry( + crc32=binascii.crc32(packed) & 0xFFFFFFFF, + object_type=object_type, + offset=entry_start, + oid=oid, + packed_size=len(packed), + payload_sha256=hashlib.sha256(payload).hexdigest(), + size=len(payload), + ) + ) + if offset != pack_end: + raise VerificationError("pack has trailing bytes outside its declared entries") + return ParsedPack( + entries=tuple(entries), + trailer_sha1=trailer.hex(), + version=version, + ) + + +def _expected_fanout(oids: tuple[str, ...]) -> tuple[int, ...]: + counts = [0] * 256 + for oid in oids: + counts[int(oid[:2], 16)] += 1 + total = 0 + fanout: list[int] = [] + for count in counts: + total += count + fanout.append(total) + return tuple(fanout) + + +def parse_index(content: bytes) -> ParsedIndex: + """Parse and verify one bounded Git index v2 without invoking Git.""" + + minimum = 8 + (256 * 4) + 40 + if ( + type(content) is not bytes + or len(content) < minimum + or len(content) > MAX_PACK_BYTES + ): + raise VerificationError("index bytes are outside the reviewed bound") + if content[:4] != INDEX_MAGIC: + raise VerificationError("index signature is invalid") + version = _uint32(content, 4, label="index version") + if version != INDEX_VERSION: + raise VerificationError("index version is outside the reviewed subset") + + fanout = tuple( + _uint32(content, 8 + (bucket * 4), label="index fanout") + for bucket in range(256) + ) + if any(left > right for left, right in zip(fanout, fanout[1:])): + raise VerificationError("index fanout table is not cumulative") + count = fanout[-1] + if count < 1 or count > MAX_PACK_OBJECTS: + raise VerificationError("index object count is outside the reviewed bound") + + oid_start = 8 + (256 * 4) + crc_start = oid_start + (count * 20) + offset_start = crc_start + (count * 4) + large_start = offset_start + (count * 4) + fixed_end = large_start + 40 + if fixed_end > len(content): + raise VerificationError("index tables are truncated") + + oids = tuple( + content[oid_start + (row * 20) : oid_start + ((row + 1) * 20)].hex() + for row in range(count) + ) + if tuple(sorted(oids)) != oids or len(set(oids)) != count: + raise VerificationError("index object IDs are not unique and sorted") + if fanout != _expected_fanout(oids): + raise VerificationError("index fanout does not match its object IDs") + + crc_values = tuple( + _uint32(content, crc_start + (row * 4), label="index CRC table") + for row in range(count) + ) + offset_words = tuple( + _uint32(content, offset_start + (row * 4), label="index offset table") + for row in range(count) + ) + large_indexes = [word & 0x7FFFFFFF for word in offset_words if word & 0x80000000] + if sorted(large_indexes) != list(range(len(large_indexes))): + raise VerificationError("index large-offset references are not canonical") + checksum_start = large_start + (len(large_indexes) * 8) + if checksum_start + 40 != len(content): + raise VerificationError("index has an invalid table length") + + large_offsets = tuple( + _uint64(content, large_start + (row * 8), label="index large-offset table") + for row in range(len(large_indexes)) + ) + offsets: list[int] = [] + for word in offset_words: + if word & 0x80000000: + offsets.append(large_offsets[word & 0x7FFFFFFF]) + else: + offsets.append(word) + + pack_sha1 = content[checksum_start : checksum_start + 20].hex() + index_sha1 = content[checksum_start + 20 :].hex() + expected_index_sha1 = hashlib.sha1( + content[: checksum_start + 20], + usedforsecurity=False, + ).hexdigest() + if index_sha1 != expected_index_sha1: + raise VerificationError("index checksum does not match the preceding bytes") + + entries = tuple( + IndexEntry(crc32=crc_values[row], offset=offsets[row], oid=oids[row]) + for row in range(count) + ) + return ParsedIndex( + entries=entries, + fanout=fanout, + index_sha1=index_sha1, + pack_sha1=pack_sha1, + version=version, + ) + + +def _cross_check(pack: ParsedPack, index: ParsedIndex) -> None: + if index.pack_sha1 != pack.trailer_sha1: + raise VerificationError("index does not bind the verified pack checksum") + pack_entries = {entry.oid: entry for entry in pack.entries} + if len(pack_entries) != len(index.entries): + raise VerificationError("pack and index object counts differ") + for indexed in index.entries: + packed = pack_entries.get(indexed.oid) + if packed is None: + raise VerificationError("index references an object absent from the pack") + if indexed.offset != packed.offset or indexed.crc32 != packed.crc32: + raise VerificationError("index offset or CRC does not match the pack entry") + + +def _build_and_verify_pack(runner: _GitRunner, private: Path) -> PackReport: + runner.initialize() + expected_payloads: dict[str, bytes] = {} + labels_by_oid: dict[str, str] = {} + for label, payload in PACK_BLOBS: + oid = _store_blob(runner, payload) + expected_payloads[oid] = payload + labels_by_oid[oid] = label + if len(expected_payloads) != len(PACK_BLOBS): + raise VerificationError("fixed pack fixture contains duplicate objects") + + ordered_oids = tuple(sorted(expected_payloads)) + prefix = private / "fixture" + result = runner.run( + "pack-objects", + "--window=0", + "--depth=0", + "--compression=0", + "--no-reuse-delta", + "--no-reuse-object", + os.fspath(prefix), + stdin=("".join(f"{oid}\n" for oid in ordered_oids)).encode("ascii"), + ) + if result.stderr: + raise VerificationError("git pack-objects produced unexpected diagnostics") + try: + pack_name = result.stdout.decode("ascii").strip() + except UnicodeDecodeError as exc: + raise VerificationError("git pack-objects returned a non-ASCII checksum") from exc + if ( + len(pack_name) != 40 + or any(character not in "0123456789abcdef" for character in pack_name) + ): + raise VerificationError("git pack-objects returned an invalid checksum") + + pack_path = Path(f"{prefix}-{pack_name}.pack") + index_path = Path(f"{prefix}-{pack_name}.idx") + pack_bytes = _read_regular_file(pack_path, label="generated pack") + index_bytes = _read_regular_file(index_path, label="generated index") + parsed_pack = parse_pack(pack_bytes) + parsed_index = parse_index(index_bytes) + _cross_check(parsed_pack, parsed_index) + if parsed_pack.trailer_sha1 != pack_name: + raise VerificationError("git pack name does not match the verified trailer") + if {entry.oid for entry in parsed_pack.entries} != set(expected_payloads): + raise VerificationError("generated pack object inventory is not exact") + for entry in parsed_pack.entries: + payload = expected_payloads[entry.oid] + if ( + entry.object_type != "blob" + or entry.size != len(payload) + or entry.payload_sha256 != hashlib.sha256(payload).hexdigest() + ): + raise VerificationError("generated pack object differs from the fixture") + + index_rows = {entry.oid: entry for entry in parsed_index.entries} + objects = [] + for entry in parsed_pack.entries: + indexed = index_rows[entry.oid] + objects.append( + { + **entry.as_dict(), + "index_crc32": f"{indexed.crc32:08x}", + "index_offset": indexed.offset, + "label": labels_by_oid[entry.oid], + } + ) + + nonzero_buckets = [ + { + "cumulative": parsed_index.fanout[bucket], + "prefix": f"{bucket:02x}", + "range_start": 0 if bucket == 0 else parsed_index.fanout[bucket - 1], + } + for bucket in range(256) + if ( + parsed_index.fanout[bucket] + != (0 if bucket == 0 else parsed_index.fanout[bucket - 1]) + ) + ] + payload: dict[str, object] = { + "checks": { + "all_fixture_objects_present": True, + "delta_entries_absent": True, + "index_checksum_verified": True, + "index_crc32_matches_pack": True, + "index_fanout_matches_sorted_oids": True, + "index_offsets_match_pack": True, + "pack_trailer_verified": True, + }, + "command_trace": list(runner.trace), + "fixture": { + "object_count": len(PACK_BLOBS), + "objects": [ + { + "label": label, + "oid": git_object_oid("blob", body), + "payload_sha256": hashlib.sha256(body).hexdigest(), + "size": len(body), + } + for label, body in PACK_BLOBS + ], + }, + "index": { + "bytes": len(index_bytes), + "index_sha1": parsed_index.index_sha1, + "nonzero_fanout_buckets": nonzero_buckets, + "pack_sha1": parsed_index.pack_sha1, + "sha256": hashlib.sha256(index_bytes).hexdigest(), + "version": parsed_index.version, + }, + "object_format": "sha1", + "objects_in_pack_order": objects, + "pack": { + "bytes": len(pack_bytes), + "delta_count": 0, + "object_count": len(parsed_pack.entries), + "sha256": hashlib.sha256(pack_bytes).hexdigest(), + "trailer_sha1": parsed_pack.trailer_sha1, + "version": parsed_pack.version, + }, + "scope": { + "authentication_claim": False, + "delta_entries_supported": False, + "fixture_kind": "three deterministic synthetic blobs", + "git_pack_objects_executed": True, + "network_required": False, + }, + "schema_version": PACK_SCHEMA_VERSION, + } + receipt = hashlib.sha256(_canonical_json(payload)).hexdigest() + return PackReport(payload=_deep_freeze(payload), receipt_sha256=receipt) + + +def run_pack_lab(root: Path | str = ".") -> PackReport: + """Build and verify a real pack/index pair below ``root``, then remove it.""" + + try: + validated_root = _validate_root(Path(root)) + git = _find_git() + with tempfile.TemporaryDirectory( + prefix=".git-pack-index-lab-", + dir=validated_root, + ) as private_name: + workspace = _new_workspace(validated_root, Path(private_name)) + runner = _GitRunner(git, workspace) + return _build_and_verify_pack(runner, workspace.private) + except LabError: + raise + except (OSError, TypeError, ValueError) as exc: + raise LabError("isolated pack lab setup failed") from exc + + +__all__ = [ + "INDEX_VERSION", + "MAX_OBJECT_BYTES", + "MAX_PACK_BYTES", + "MAX_PACK_OBJECTS", + "PACK_SCHEMA_VERSION", + "PACK_VERSION", + "IndexEntry", + "PackEntry", + "PackReport", + "ParsedIndex", + "ParsedPack", + "parse_index", + "parse_pack", + "run_pack_lab", +] diff --git a/tests/test_evidence.py b/tests/test_evidence.py index aed73dc..7b1c48e 100644 --- a/tests/test_evidence.py +++ b/tests/test_evidence.py @@ -244,7 +244,7 @@ def test_readme_baseline_matches_current_evidence(self) -> None: self.assertIn("**57 isolated Git invocations**", readme) self.assertIn(f"report receipt `{receipt_label}`", readme) self.assertIn(f"screenshot SHA-256 `{screenshot_label}`", readme) - self.assertIn("10 local subcommands are allow-listed", readme) + self.assertIn("11 local subcommands are allow-listed", readme) if __name__ == "__main__": diff --git a/tests/test_pack_evidence.py b/tests/test_pack_evidence.py new file mode 100644 index 0000000..ccb03e7 --- /dev/null +++ b/tests/test_pack_evidence.py @@ -0,0 +1,236 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import re +import unittest + +from tools import generate_pack_evidence as evidence + +ROOT = Path(__file__).resolve().parents[1] + + +class PackEvidenceTests(unittest.TestCase): + def test_expected_generated_inventory_is_exact(self) -> None: + generated = evidence.build_artifacts(allow_missing_screenshot=False) + self.assertEqual( + set(generated), + { + evidence.EVIDENCE_PATH, + evidence.VERIFY_PATH, + evidence.INSPECT_PATH, + evidence.REPORT_PATH, + evidence.MANIFEST_PATH, + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + }, + ) + + def test_checked_in_generated_files_are_current(self) -> None: + generated = evidence.build_artifacts(allow_missing_screenshot=False) + for path, expected in generated.items(): + with self.subTest(path=path.as_posix()): + self.assertEqual((ROOT / path).read_bytes(), expected) + + def test_compact_pretty_and_verify_outputs_share_one_receipt(self) -> None: + compact = json.loads((ROOT / evidence.EVIDENCE_PATH).read_text()) + pretty = json.loads((ROOT / evidence.INSPECT_PATH).read_text()) + verify = (ROOT / evidence.VERIFY_PATH).read_text() + self.assertEqual(compact, pretty) + self.assertIn( + f"receipt_sha256={compact['receipt']['sha256']}", + verify, + ) + canonical = json.dumps( + compact, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + b"\n" + self.assertEqual((ROOT / evidence.EVIDENCE_PATH).read_bytes(), canonical) + + def test_two_fresh_evidence_builds_are_byte_identical(self) -> None: + first = evidence.build_artifacts(allow_missing_screenshot=False) + second = evidence.build_artifacts(allow_missing_screenshot=False) + self.assertEqual(first, second) + + def test_manifest_binds_artifacts_capture_and_sources(self) -> None: + manifest = json.loads((ROOT / evidence.MANIFEST_PATH).read_text()) + rows = {row["path"]: row for row in manifest["artifacts"]} + expected = { + evidence.EVIDENCE_PATH, + evidence.VERIFY_PATH, + evidence.INSPECT_PATH, + evidence.REPORT_PATH, + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + evidence.SCREENSHOT_PATH, + evidence.RENDERED_DOM_PATH, + evidence.ATTESTATION_PATH, + } + self.assertEqual(set(rows), {path.as_posix() for path in expected}) + for path in expected: + content = (ROOT / path).read_bytes() + row = rows[path.as_posix()] + self.assertEqual(row["size"], len(content)) + self.assertEqual(row["sha256"], hashlib.sha256(content).hexdigest()) + self.assertEqual(manifest["capture"]["status"], "attested") + sources = {row["path"]: row for row in manifest["sources"]} + self.assertEqual( + set(sources), + { + "git_dag_lab/pack.py", + "git_dag_lab/cli.py", + "tools/generate_pack_evidence.py", + "tools/capture_pack_report.sh", + }, + ) + for relative, row in sources.items(): + content = (ROOT / relative).read_bytes() + self.assertEqual(row["size"], len(content)) + self.assertEqual(row["sha256"], hashlib.sha256(content).hexdigest()) + + def test_svg_visuals_are_accessible_and_receipt_bound(self) -> None: + document = json.loads((ROOT / evidence.EVIDENCE_PATH).read_text()) + receipt = document["receipt"]["sha256"] + for path in ( + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + ): + text = (ROOT / path).read_text() + with self.subTest(path=path.as_posix()): + self.assertIn("", 1)[1].split("", 1)[0] + ) + self.assertEqual( + metadata["source"], + evidence.EVIDENCE_PATH.as_posix(), + ) + self.assertEqual(metadata["report_receipt_sha256"], receipt) + self.assertNotIn("/home/", text) + self.assertNotIn("github.com/", text) + + def test_offline_report_has_no_executable_or_external_content(self) -> None: + report = (ROOT / evidence.REPORT_PATH).read_text() + self.assertNotIn(" None: + screenshot = (ROOT / evidence.SCREENSHOT_PATH).read_bytes() + self.assertEqual(evidence._parse_pack_png(screenshot), evidence.PNG_DIMENSIONS) + attestation = json.loads((ROOT / evidence.ATTESTATION_PATH).read_text()) + payload = attestation["attestation"] + canonical = json.dumps( + payload, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + self.assertEqual( + attestation["receipt"]["sha256"], + hashlib.sha256(canonical).hexdigest(), + ) + self.assertEqual(payload["isolation"]["network"], "none") + self.assertEqual( + payload["input"]["report"]["report_receipt_sha256"], + json.loads((ROOT / evidence.EVIDENCE_PATH).read_text())["receipt"][ + "sha256" + ], + ) + + def test_public_pack_evidence_has_no_host_or_secret_markers(self) -> None: + paths = ( + evidence.EVIDENCE_PATH, + evidence.VERIFY_PATH, + evidence.INSPECT_PATH, + evidence.REPORT_PATH, + evidence.MANIFEST_PATH, + evidence.RENDERED_DOM_PATH, + evidence.ATTESTATION_PATH, + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + ) + forbidden = ( + "/home/", + "@gmail.com", + "AKIA", + "ghp_", + "github_pat_", + "OPENAI_API_KEY", + ) + for path in paths: + text = (ROOT / path).read_text(errors="strict") + with self.subTest(path=path.as_posix()): + for marker in forbidden: + self.assertNotIn(marker.lower(), text.lower()) + + def test_document_validator_rejects_claim_and_integrity_drift(self) -> None: + document = json.loads((ROOT / evidence.EVIDENCE_PATH).read_text()) + verify = (ROOT / evidence.VERIFY_PATH).read_bytes() + mutations = [] + + missing_check = deepcopy(document) + del missing_check["report"]["checks"]["pack_trailer_verified"] + mutations.append(missing_check) + + delta_claim = deepcopy(document) + delta_claim["report"]["scope"]["delta_entries_supported"] = True + mutations.append(delta_claim) + + authentication_claim = deepcopy(document) + authentication_claim["report"]["scope"]["authentication_claim"] = True + mutations.append(authentication_claim) + + changed_crc = deepcopy(document) + changed_crc["report"]["objects_in_pack_order"][0]["index_crc32"] = "00000000" + mutations.append(changed_crc) + + for mutation in mutations: + with self.assertRaises(evidence.EvidenceError): + evidence._validate_document(mutation, verify) + + def test_readme_exposes_real_pack_workflow_and_visuals(self) -> None: + readme = (ROOT / "README.md").read_text() + for path in ( + evidence.SCREENSHOT_PATH, + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + ): + self.assertIn(path.as_posix(), readme) + self.assertIn("python3 -m git_dag_lab pack-verify", readme) + self.assertIn("python3 -m git_dag_lab pack-inspect", readme) + self.assertIn("Delta", readme) + self.assertRegex(readme, r"\*\*[0-9]+ tests\*\*") + + def test_capture_toolchain_is_digest_pinned_and_network_disabled(self) -> None: + script = (ROOT / "tools/capture_pack_report.sh").read_text() + self.assertRegex(script, r"mcr\.microsoft\.com/playwright@sha256:[0-9a-f]{64}") + self.assertIn("--network none", script) + self.assertIn("--pull=never", script) + self.assertIn("--read-only", script) + self.assertNotRegex(script, re.compile(r"(?m)^\s*curl\b|\bwget\b")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pack_lab.py b/tests/test_pack_lab.py new file mode 100644 index 0000000..8a4c843 --- /dev/null +++ b/tests/test_pack_lab.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import binascii +import hashlib +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +import zlib +from unittest import mock + +from git_dag_lab import pack as pack_module +from git_dag_lab.cli import main +from git_dag_lab.lab import VerificationError, git_object_oid +from git_dag_lab.pack import ( + INDEX_MAGIC, + INDEX_VERSION, + MAX_PACK_BYTES, + PACK_MAGIC, + PACK_SCHEMA_VERSION, + PACK_VERSION, + parse_index, + parse_pack, + run_pack_lab, +) + + +def _entry_header(type_code: int, size: int) -> bytes: + first = (type_code << 4) | (size & 0x0F) + size >>= 4 + output = bytearray((first,)) + if size: + output[0] |= 0x80 + while size: + current = size & 0x7F + size >>= 7 + if size: + current |= 0x80 + output.append(current) + return bytes(output) + + +def _resign_pack(prefix: bytes) -> bytes: + return prefix + hashlib.sha1(prefix, usedforsecurity=False).digest() + + +def _build_pack(payloads: tuple[bytes, ...]) -> tuple[bytes, list[dict[str, int | str]]]: + body = bytearray(PACK_MAGIC) + body.extend(PACK_VERSION.to_bytes(4, "big")) + body.extend(len(payloads).to_bytes(4, "big")) + rows: list[dict[str, int | str]] = [] + for payload in payloads: + offset = len(body) + packed = _entry_header(3, len(payload)) + zlib.compress(payload, level=9) + body.extend(packed) + rows.append( + { + "crc32": binascii.crc32(packed) & 0xFFFFFFFF, + "offset": offset, + "oid": git_object_oid("blob", payload), + } + ) + return _resign_pack(bytes(body)), rows + + +def _resign_index(prefix: bytes) -> bytes: + return prefix + hashlib.sha1(prefix, usedforsecurity=False).digest() + + +def _build_index( + rows: list[dict[str, int | str]], + pack_sha1: str, +) -> bytes: + ordered = sorted(rows, key=lambda row: str(row["oid"])) + counts = [0] * 256 + for row in ordered: + counts[int(str(row["oid"])[:2], 16)] += 1 + fanout: list[int] = [] + total = 0 + for count in counts: + total += count + fanout.append(total) + + body = bytearray(INDEX_MAGIC) + body.extend(INDEX_VERSION.to_bytes(4, "big")) + for value in fanout: + body.extend(value.to_bytes(4, "big")) + for row in ordered: + body.extend(bytes.fromhex(str(row["oid"]))) + for row in ordered: + body.extend(int(row["crc32"]).to_bytes(4, "big")) + for row in ordered: + body.extend(int(row["offset"]).to_bytes(4, "big")) + body.extend(bytes.fromhex(pack_sha1)) + return _resign_index(bytes(body)) + + +class PackParserTests(unittest.TestCase): + def setUp(self) -> None: + self.payloads = (b"fixture\n", b"\x00binary\xff\n") + self.pack_bytes, self.rows = _build_pack(self.payloads) + + def test_decodes_real_envelopes_and_verifies_trailer(self) -> None: + parsed = parse_pack(self.pack_bytes) + self.assertEqual(parsed.version, PACK_VERSION) + self.assertEqual(len(parsed.entries), 2) + self.assertEqual( + {entry.oid for entry in parsed.entries}, + {git_object_oid("blob", payload) for payload in self.payloads}, + ) + self.assertEqual( + parsed.trailer_sha1, + hashlib.sha1(self.pack_bytes[:-20], usedforsecurity=False).hexdigest(), + ) + for entry, payload in zip(parsed.entries, self.payloads, strict=True): + self.assertEqual(entry.object_type, "blob") + self.assertEqual(entry.size, len(payload)) + self.assertEqual(entry.payload_sha256, hashlib.sha256(payload).hexdigest()) + self.assertGreater(entry.packed_size, 1) + + def test_rejects_non_bytes_and_outer_boundary_drift(self) -> None: + cases = ( + bytearray(self.pack_bytes), + b"", + self.pack_bytes[:31], + b"x" * (MAX_PACK_BYTES + 1), + ) + for content in cases: + with self.subTest(size=len(content)), self.assertRaises(VerificationError): + parse_pack(content) # type: ignore[arg-type] + + def test_rejects_signature_version_and_count_drift(self) -> None: + bad_signature = b"FAIL" + self.pack_bytes[4:] + bad_version = bytearray(self.pack_bytes) + bad_version[4:8] = (3).to_bytes(4, "big") + bad_count = bytearray(self.pack_bytes) + bad_count[8:12] = (0).to_bytes(4, "big") + for content in ( + bad_signature, + _resign_pack(bytes(bad_version[:-20])), + _resign_pack(bytes(bad_count[:-20])), + ): + with self.assertRaises(VerificationError): + parse_pack(content) + + def test_rejects_delta_type_size_zlib_and_trailer_mutations(self) -> None: + delta = bytearray(self.pack_bytes[:-20]) + delta[12] = (delta[12] & 0x8F) | 0x60 + + wrong_size = bytearray(self.pack_bytes[:-20]) + wrong_size[12] = (wrong_size[12] & 0xF0) | ((len(self.payloads[0]) + 1) & 0x0F) + + corrupt_zlib = bytearray(self.pack_bytes[:-20]) + corrupt_zlib[14] ^= 0xFF + + bad_trailer = bytearray(self.pack_bytes) + bad_trailer[-1] ^= 0x01 + + for content in ( + _resign_pack(bytes(delta)), + _resign_pack(bytes(wrong_size)), + _resign_pack(bytes(corrupt_zlib)), + bytes(bad_trailer), + ): + with self.assertRaises(VerificationError): + parse_pack(content) + + def test_rejects_declared_extra_object_and_trailing_body_bytes(self) -> None: + extra_object = bytearray(self.pack_bytes[:-20]) + extra_object[8:12] = (3).to_bytes(4, "big") + trailing = self.pack_bytes[:-20] + b"\x00" + for content in ( + _resign_pack(bytes(extra_object)), + _resign_pack(trailing), + ): + with self.assertRaises(VerificationError): + parse_pack(content) + + +class IndexParserTests(unittest.TestCase): + def setUp(self) -> None: + self.pack_bytes, self.rows = _build_pack((b"alpha\n", b"omega\n")) + self.pack = parse_pack(self.pack_bytes) + self.index_bytes = _build_index(self.rows, self.pack.trailer_sha1) + + def test_decodes_fanout_rows_and_both_checksums(self) -> None: + parsed = parse_index(self.index_bytes) + self.assertEqual(parsed.version, INDEX_VERSION) + self.assertEqual(parsed.pack_sha1, self.pack.trailer_sha1) + self.assertEqual(parsed.fanout[-1], len(self.rows)) + self.assertEqual( + tuple(entry.oid for entry in parsed.entries), + tuple(sorted(str(row["oid"]) for row in self.rows)), + ) + self.assertEqual( + parsed.index_sha1, + hashlib.sha1(self.index_bytes[:-20], usedforsecurity=False).hexdigest(), + ) + pack_module._cross_check(self.pack, parsed) + + def test_rejects_signature_version_checksum_and_length_drift(self) -> None: + bad_version = bytearray(self.index_bytes) + bad_version[4:8] = (3).to_bytes(4, "big") + bad_checksum = bytearray(self.index_bytes) + bad_checksum[-1] ^= 0x01 + trailing = self.index_bytes[:-20] + b"\x00" + for content in ( + b"FAIL" + self.index_bytes[4:], + _resign_index(bytes(bad_version[:-20])), + bytes(bad_checksum), + _resign_index(trailing), + ): + with self.assertRaises(VerificationError): + parse_index(content) + + def test_rejects_fanout_oid_and_large_offset_drift(self) -> None: + oid_start = 8 + (256 * 4) + wrong_oid = bytearray(self.index_bytes[:-20]) + wrong_oid[oid_start] ^= 0x01 + + count = len(self.rows) + offset_start = oid_start + (count * 20) + (count * 4) + bad_large_offset = bytearray(self.index_bytes[:-20]) + bad_large_offset[offset_start : offset_start + 4] = ( + 0x80000001 + ).to_bytes(4, "big") + + for content in ( + _resign_index(bytes(wrong_oid)), + _resign_index(bytes(bad_large_offset)), + ): + with self.assertRaises(VerificationError): + parse_index(content) + + def test_cross_check_rejects_crc_offset_inventory_and_pack_binding_drift(self) -> None: + parsed = parse_index(self.index_bytes) + + changed_crc = bytearray(self.index_bytes[:-20]) + count = len(self.rows) + oid_start = 8 + (256 * 4) + crc_start = oid_start + (count * 20) + changed_crc[crc_start + 3] ^= 0x01 + + changed_offset = bytearray(self.index_bytes[:-20]) + offset_start = crc_start + (count * 4) + changed_offset[offset_start + 3] ^= 0x01 + + changed_pack = bytearray(self.index_bytes[:-20]) + changed_pack[-1] ^= 0x01 + + for content in ( + _resign_index(bytes(changed_crc)), + _resign_index(bytes(changed_offset)), + _resign_index(bytes(changed_pack)), + ): + with self.assertRaises(VerificationError): + pack_module._cross_check(self.pack, parse_index(content)) + self.assertEqual(len(parsed.entries), 2) + + +class PackRuntimeTests(unittest.TestCase): + def test_real_pack_and_index_are_deterministic_and_leave_no_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + first = run_pack_lab(root) + self.assertEqual(list(root.iterdir()), []) + second = run_pack_lab(root) + self.assertEqual(list(root.iterdir()), []) + + self.assertEqual(first.to_json(), second.to_json()) + report = first.document["report"] + self.assertEqual(report["schema_version"], PACK_SCHEMA_VERSION) + self.assertEqual(report["pack"]["version"], PACK_VERSION) + self.assertEqual(report["index"]["version"], INDEX_VERSION) + self.assertEqual(report["pack"]["object_count"], 3) + self.assertEqual(report["pack"]["delta_count"], 0) + self.assertEqual( + report["command_trace"], + ["init", "hash-object", "hash-object", "hash-object", "pack-objects"], + ) + self.assertTrue(all(report["checks"].values())) + self.assertFalse(report["scope"]["delta_entries_supported"]) + self.assertFalse(report["scope"]["authentication_claim"]) + + def test_receipt_binds_the_complete_payload(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + result = run_pack_lab(Path(temporary)) + document = result.document + canonical = json.dumps( + document["report"], + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + self.assertEqual( + document["receipt"]["sha256"], + hashlib.sha256(canonical).hexdigest(), + ) + self.assertEqual(json.loads(result.to_json()), document) + + def test_pack_cli_exposes_receipt_and_canonical_document(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + stdout = io.StringIO() + stderr = io.StringIO() + self.assertEqual( + main( + ["pack-verify"], + root=root, + stdout=stdout, + stderr=stderr, + ), + 0, + ) + self.assertEqual(stderr.getvalue(), "") + self.assertRegex( + stdout.getvalue(), + rf"^PASS {PACK_SCHEMA_VERSION} objects=3 .* receipt_sha256=[0-9a-f]{{64}}\n$", + ) + + inspect_output = io.StringIO() + self.assertEqual( + main( + ["pack-inspect", "--compact"], + root=root, + stdout=inspect_output, + stderr=io.StringIO(), + ), + 0, + ) + inspected = json.loads(inspect_output.getvalue()) + self.assertEqual(inspected["report"]["schema_version"], PACK_SCHEMA_VERSION) + + def test_pack_cli_returns_sanitized_lab_errors(self) -> None: + stdout = io.StringIO() + stderr = io.StringIO() + with mock.patch( + "git_dag_lab.cli.run_pack_lab", + side_effect=VerificationError("reviewed failure"), + ): + self.assertEqual( + main( + ["pack-verify"], + root=Path.cwd(), + stdout=stdout, + stderr=stderr, + ), + 1, + ) + self.assertEqual(stdout.getvalue(), "") + self.assertEqual(stderr.getvalue(), "ERROR git-dag-lab: reviewed failure\n") + + +class PackFileBoundaryTests(unittest.TestCase): + def test_reads_one_regular_single_link_file(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "pack" + path.write_bytes(b"bounded") + self.assertEqual( + pack_module._read_regular_file(path, label="fixture"), + b"bounded", + ) + + @unittest.skipUnless(hasattr(os, "symlink"), "symlink support is required") + def test_rejects_symlink_and_hardlink_inputs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = root / "target" + target.write_bytes(b"bounded") + symlink = root / "symlink" + symlink.symlink_to(target.name) + hardlink = root / "hardlink" + os.link(target, hardlink) + for path in (symlink, target, hardlink): + with self.subTest(path=path.name), self.assertRaises( + VerificationError + ): + pack_module._read_regular_file(path, label="fixture") + + def test_rejects_empty_and_oversized_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + empty = root / "empty" + empty.write_bytes(b"") + oversized = root / "oversized" + with oversized.open("wb") as stream: + stream.truncate(MAX_PACK_BYTES + 1) + for path in (empty, oversized): + with self.assertRaises(VerificationError): + pack_module._read_regular_file(path, label="fixture") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/capture_pack_report.sh b/tools/capture_pack_report.sh new file mode 100755 index 0000000..9b09f7c --- /dev/null +++ b/tools/capture_pack_report.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 0 ]]; then + echo "usage: $0" >&2 + exit 2 +fi + +readonly image='mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c' +readonly browser='/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell' +readonly browser_version='Chromium 140.0.7339.186' +readonly browser_sha256='003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d' + +repo_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd -P) +demo_root="$repo_root/docs/demo/git-pack-index-v1" +report_path="$demo_root/report.html" +output_path="$repo_root/docs/assets/git-pack-report.png" +rendered_dom_path="$demo_root/rendered-dom.html" +attestation_path="$demo_root/capture-attestation.json" + +if ! command -v docker >/dev/null 2>&1; then + echo "capture error: Docker is required" >&2 + exit 1 +fi +if [[ $(id -u) -eq 0 || $(id -g) -eq 0 ]]; then + echo "capture error: refusing to run Chromium as root" >&2 + exit 1 +fi + +python3 -B "$repo_root/tools/generate_pack_evidence.py" --write --allow-missing-screenshot >/dev/null +python3 -B "$repo_root/tools/generate_pack_evidence.py" --check --allow-missing-screenshot >/dev/null + +for managed_path in \ + "$repo_root/docs" \ + "$repo_root/docs/demo" \ + "$demo_root" \ + "$report_path" \ + "$repo_root/docs/assets"; do + if [[ ! -e "$managed_path" || $(realpath -e -- "$managed_path") != "$managed_path" ]]; then + echo "capture error: managed evidence path is missing or contains a symlink" >&2 + exit 1 + fi +done +if [[ ! -f "$report_path" || -L "$report_path" ]]; then + echo "capture error: offline report is unsafe" >&2 + exit 1 +fi +for destination in "$output_path" "$rendered_dom_path" "$attestation_path"; do + if [[ -L "$destination" || ( -e "$destination" && ! -f "$destination" ) ]]; then + echo "capture error: managed capture destination is unsafe" >&2 + exit 1 + fi + if [[ -e "$destination" && $(stat -c '%h' -- "$destination") -ne 1 ]]; then + echo "capture error: managed capture destination has multiple hard links" >&2 + exit 1 + fi +done + +report_sha_before=$(sha256sum "$report_path" | awk '{print $1}') +script_sha_before=$(sha256sum "$repo_root/tools/capture_pack_report.sh" | awk '{print $1}') + +observed_digest=$(docker image inspect --format '{{index .RepoDigests 0}}' "$image") +if [[ "$observed_digest" != "$image" ]]; then + echo "capture error: cached container digest is not exact" >&2 + exit 1 +fi +observed_architecture=$(docker image inspect --format '{{.Architecture}}' "$image") +if [[ "$observed_architecture" != "amd64" ]]; then + echo "capture error: cached container architecture is not amd64" >&2 + exit 1 +fi + +uid=$(id -u) +gid=$(id -g) +readonly uid +readonly gid +common=( + --rm + --pull=never + --platform linux/amd64 + --network none + --user "$uid:$gid" + --read-only + --cap-drop ALL + --security-opt no-new-privileges + --pids-limit 256 + --memory 768m + --memory-swap 768m + --cpus 1 + --ulimit nofile=1024:1024 + --ulimit core=0:0 + --tmpfs '/tmp:rw,nosuid,nodev,noexec,size=256m,mode=1777' + --tmpfs '/dev/shm:rw,nosuid,nodev,noexec,size=256m,mode=1777' + --env HOME=/tmp + --env XDG_CACHE_HOME=/tmp/cache + --env XDG_CONFIG_HOME=/tmp/config + --env LANG=C.UTF-8 + --env LC_ALL=C.UTF-8 + --env TZ=UTC +) + +observed_hash=$(docker run "${common[@]}" --entrypoint /usr/bin/sha256sum \ + "$image" "$browser" | awk '{print $1}') +if [[ "$observed_hash" != "$browser_sha256" ]]; then + echo "capture error: Chromium binary hash is not exact" >&2 + exit 1 +fi +observed_version=$(docker run "${common[@]}" --entrypoint "$browser" \ + "$image" --version) +if [[ "$observed_version" != "$browser_version" ]]; then + echo "capture error: Chromium version is not exact" >&2 + exit 1 +fi + +temporary_root=$(mktemp -d "$repo_root/.git-pack-report-capture.XXXXXX") +cidfile="$temporary_root/container.cid" +cleanup() { + if [[ -f "$cidfile" && ! -L "$cidfile" ]]; then + container_id=$(<"$cidfile") + if [[ "$container_id" =~ ^[0-9a-f]{64}$ ]]; then + docker rm -f "$container_id" >/dev/null 2>&1 || true + fi + fi + if [[ -d "$temporary_root" ]]; then + find "$temporary_root" -depth -delete + fi +} +trap cleanup EXIT INT TERM +mkdir "$temporary_root/output" +dom_path="$temporary_root/rendered-dom.html" +capture_log="$temporary_root/capture.stderr" + +capture=( + docker run + "${common[@]}" + --cidfile "$cidfile" + --mount "type=bind,src=$demo_root,dst=/demo,readonly" + --mount "type=bind,src=$temporary_root/output,dst=/output" + --workdir /demo + --entrypoint "$browser" + "$image" + --headless + --no-sandbox + --disable-background-networking + --disable-breakpad + --disable-component-update + --disable-default-apps + --disable-extensions + '--disable-features=OptimizationHints,Translate' + --disable-sync + --force-color-profile=srgb + --force-device-scale-factor=1 + --hide-scrollbars + '--host-resolver-rules=MAP * ~NOTFOUND' + --lang=en-US + --metrics-recording-only + --no-first-run + --no-pings + --password-store=basic + --run-all-compositor-stages-before-draw + --safebrowsing-disable-auto-update + --virtual-time-budget=1000 + '--window-size=1440,1500' + --dump-dom + --screenshot=/output/git-pack-report.png + file:///demo/report.html +) + +if ! timeout --signal=TERM --kill-after=5s 45s \ + "${capture[@]}" >"$dom_path" 2>"$capture_log"; then + echo "capture error: isolated Chromium capture failed" >&2 + sed -n '1,12p' "$capture_log" >&2 + exit 1 +fi + +python3 - "$repo_root" "$repo_root/evidence/git-pack-index-v1.json" "$dom_path" "$temporary_root/output" <<'PY' +from pathlib import Path +import json +import sys + +repo_root = Path(sys.argv[1]) +evidence_path = Path(sys.argv[2]) +dom_path = Path(sys.argv[3]) +output = Path(sys.argv[4]) +sys.path.insert(0, str(repo_root)) +from tools.generate_pack_evidence import _parse_pack_png + +document = json.loads(evidence_path.read_text(encoding="utf-8")) +receipt = document["receipt"]["sha256"] +dom = dom_path.read_text(encoding="utf-8") +required = ( + '
/dev/null +if [[ $(sha256sum "$report_path" | awk '{print $1}') != "$report_sha_before" ]]; then + echo "capture error: offline report changed during capture" >&2 + exit 1 +fi +if [[ $(sha256sum "$repo_root/tools/capture_pack_report.sh" | awk '{print $1}') != "$script_sha_before" ]]; then + echo "capture error: capture script changed during execution" >&2 + exit 1 +fi + +chmod 0644 "$temporary_root/output/git-pack-report.png" +chmod 0644 "$dom_path" +mv -f -- "$temporary_root/output/git-pack-report.png" "$output_path" +mv -f -- "$dom_path" "$rendered_dom_path" + +python3 - "$repo_root" "$report_path" "$rendered_dom_path" "$output_path" "$attestation_path" <<'PY' +from pathlib import Path +import hashlib +import json +import os +import sys +import tempfile + +root = Path(sys.argv[1]) +report_path = Path(sys.argv[2]) +dom_path = Path(sys.argv[3]) +screenshot_path = Path(sys.argv[4]) +attestation_path = Path(sys.argv[5]) + +def sha256(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + +def row(path: Path, relative: str) -> dict[str, object]: + content = path.read_bytes() + return {"path": relative, "sha256": sha256(content), "size": len(content)} + +report = report_path.read_bytes() +dom = dom_path.read_bytes() +screenshot = screenshot_path.read_bytes() +script_path = root / "tools/capture_pack_report.sh" +script = script_path.read_bytes() +evidence = json.loads((root / "evidence/git-pack-index-v1.json").read_text(encoding="utf-8")) + +payload = { + "browser": { + "binary_path": "/ms-playwright/chromium_headless_shell-1193/chrome-linux/headless_shell", + "sha256": "003728e0b77eb9d52e4d258594bd55ce22ecd245eb6d3b6858fbd844c901ad7d", + "version": "Chromium 140.0.7339.186", + }, + "container": { + "architecture": "amd64", + "image": "mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c", + }, + "input": { + "report": { + **row(report_path, "docs/demo/git-pack-index-v1/report.html"), + "report_receipt_sha256": evidence["receipt"]["sha256"], + } + }, + "isolation": { + "capabilities": "all-dropped", + "demo_mount": "read-only", + "network": "none", + "no_new_privileges": True, + "pull": "never", + "root_filesystem": "read-only", + "user": "caller-nonroot", + }, + "outputs": { + "rendered_dom": row(dom_path, "docs/demo/git-pack-index-v1/rendered-dom.html"), + "screenshot": { + **row(screenshot_path, "docs/assets/git-pack-report.png"), + "height": 1500, + "width": 1440, + }, + }, + "schema_version": "git-pack-browser-capture-attestation/v1", + "script": row(script_path, "tools/capture_pack_report.sh"), + "viewport": {"device_scale_factor": 1, "height": 1500, "width": 1440}, +} +canonical = json.dumps( + payload, ensure_ascii=True, separators=(",", ":"), sort_keys=True +).encode("utf-8") +document = { + "attestation": payload, + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": sha256(canonical), + }, +} +content = (json.dumps(document, ensure_ascii=True, indent=2, sort_keys=True) + "\n").encode("utf-8") +temporary_path: Path | None = None +try: + with tempfile.NamedTemporaryFile( + mode="wb", prefix=".capture-attestation-", dir=attestation_path.parent, delete=False + ) as temporary: + temporary.write(content) + temporary.flush() + os.fsync(temporary.fileno()) + temporary_path = Path(temporary.name) + os.chmod(temporary_path, 0o644) + os.replace(temporary_path, attestation_path) + temporary_path = None +finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) +PY + +python3 -B "$repo_root/tools/generate_pack_evidence.py" --write >/dev/null +python3 -B "$repo_root/tools/generate_pack_evidence.py" --check >/dev/null +sha256sum "$output_path" diff --git a/tools/generate_pack_evidence.py b/tools/generate_pack_evidence.py new file mode 100755 index 0000000..ac669e2 --- /dev/null +++ b/tools/generate_pack_evidence.py @@ -0,0 +1,906 @@ +#!/usr/bin/env python3 +"""Generate and verify source-bound Git pack/index evidence.""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +import hashlib +import html +import json +import os +from pathlib import Path +import re +import struct +import subprocess +import sys +from typing import Any +import zlib + +ROOT = Path(__file__).resolve().parents[1] +if os.fspath(ROOT) not in sys.path: + sys.path.insert(0, os.fspath(ROOT)) + +from tools.generate_evidence import ( + BROWSER_PATH, + BROWSER_SHA256, + BROWSER_VERSION, + CONTAINER_IMAGE, + EvidenceError, + MAX_PNG_BYTES, + _artifact_row, + _check_artifacts, + _json_bytes, + _safe_capture_bytes, + _sha256, + _source_row, + _svg_header as _base_svg_header, + _write_artifacts, + _xml_text, +) + +EVIDENCE_PATH = Path("evidence/git-pack-index-v1.json") +DEMO_ROOT = Path("docs/demo/git-pack-index-v1") +VERIFY_PATH = DEMO_ROOT / "verify.txt" +INSPECT_PATH = DEMO_ROOT / "inspect.json" +REPORT_PATH = DEMO_ROOT / "report.html" +MANIFEST_PATH = DEMO_ROOT / "manifest.json" +RENDERED_DOM_PATH = DEMO_ROOT / "rendered-dom.html" +ATTESTATION_PATH = DEMO_ROOT / "capture-attestation.json" +LAYOUT_PATH = Path("docs/assets/git-pack-layout.svg") +FANOUT_PATH = Path("docs/assets/git-pack-fanout.svg") +INTEGRITY_PATH = Path("docs/assets/git-pack-integrity.svg") +CLI_PATH = Path("docs/assets/git-pack-cli.svg") +SCREENSHOT_PATH = Path("docs/assets/git-pack-report.png") +PNG_DIMENSIONS = (1440, 1500) +SHA1_RE = re.compile(r"^[0-9a-f]{40}$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +@dataclass(frozen=True, slots=True) +class PackCapture: + screenshot: bytes + rendered_dom: bytes + attestation: bytes + document: Mapping[str, Any] + + +def _pack_svg_header( + title: str, + description: str, + *, + width: int, + height: int, + receipt: str, + metadata_extra: Mapping[str, Any] | None = None, +) -> str: + """Build an SVG header whose provenance names the pack evidence source.""" + + metadata = dict(metadata_extra or {}) + metadata["source"] = EVIDENCE_PATH.as_posix() + return _base_svg_header( + title, + description, + width=width, + height=height, + receipt=receipt, + metadata_extra=metadata, + ) + + +def _parse_pack_png(content: bytes) -> tuple[int, int]: + """Validate the complete pack-report PNG before trusting image metadata.""" + + if len(content) > MAX_PNG_BYTES: + raise EvidenceError("pack browser capture exceeds the PNG size limit") + if len(content) < 57 or not content.startswith(b"\x89PNG\r\n\x1a\n"): + raise EvidenceError("pack browser capture is not a complete PNG") + + offset = 8 + chunk_index = 0 + width = height = 0 + idat_payloads: list[bytes] = [] + saw_iend = False + while offset < len(content): + if len(content) - offset < 12: + raise EvidenceError("pack PNG chunk header is truncated") + length = struct.unpack(">I", content[offset : offset + 4])[0] + chunk_type = content[offset + 4 : offset + 8] + chunk_end = offset + 12 + length + if chunk_end > len(content): + raise EvidenceError("pack PNG chunk payload is truncated") + if len(chunk_type) != 4 or not all( + 65 <= byte <= 90 or 97 <= byte <= 122 for byte in chunk_type + ): + raise EvidenceError("pack PNG chunk type is malformed") + payload = content[offset + 8 : offset + 8 + length] + expected_crc = struct.unpack(">I", content[offset + 8 + length : chunk_end])[0] + actual_crc = zlib.crc32(chunk_type + payload) & 0xFFFFFFFF + if expected_crc != actual_crc: + raise EvidenceError("pack PNG chunk CRC does not match") + + if chunk_index == 0: + if chunk_type != b"IHDR" or length != 13: + raise EvidenceError("pack PNG does not start with one 13-byte IHDR") + width, height, bit_depth, color_type, compression, filtering, interlace = ( + struct.unpack(">IIBBBBB", payload) + ) + if ( + (width, height) != PNG_DIMENSIONS + or bit_depth != 8 + or color_type != 2 + or compression != 0 + or filtering != 0 + or interlace != 0 + ): + raise EvidenceError( + "pack PNG IHDR differs from the pinned Chromium profile" + ) + elif chunk_type == b"IDAT": + idat_payloads.append(payload) + elif chunk_type != b"IEND": + raise EvidenceError( + "pack PNG contains a chunk outside the pinned Chromium profile" + ) + + offset = chunk_end + chunk_index += 1 + if chunk_type == b"IEND": + if length != 0: + raise EvidenceError("pack PNG IEND chunk is not empty") + saw_iend = True + break + + if not idat_payloads or not saw_iend: + raise EvidenceError("pack PNG lacks IDAT or terminal IEND") + if offset != len(content): + raise EvidenceError("pack PNG has trailing bytes after IEND") + + scanline_size = 1 + width * 3 + expected_decoded_size = height * scanline_size + decoder = zlib.decompressobj() + decoded = bytearray() + try: + for index, payload in enumerate(idat_payloads): + if decoder.eof: + raise EvidenceError( + "pack PNG contains IDAT data after the zlib stream ended" + ) + remaining = expected_decoded_size + 1 - len(decoded) + if remaining <= 0: + raise EvidenceError("pack PNG scanline stream exceeds the expected size") + decoded.extend(decoder.decompress(payload, remaining)) + if decoder.unconsumed_tail or decoder.unused_data: + raise EvidenceError("pack PNG zlib stream has excess compressed data") + if decoder.eof and index != len(idat_payloads) - 1: + raise EvidenceError( + "pack PNG zlib stream ended before the final IDAT" + ) + remaining = expected_decoded_size + 1 - len(decoded) + if remaining <= 0: + raise EvidenceError("pack PNG scanline stream exceeds the expected size") + decoded.extend(decoder.flush(remaining)) + except zlib.error as exc: + raise EvidenceError("pack PNG IDAT payload is not a valid zlib stream") from exc + if ( + not decoder.eof + or decoder.unused_data + or decoder.unconsumed_tail + or len(decoded) != expected_decoded_size + ): + raise EvidenceError("pack PNG scanline stream is incomplete or has the wrong size") + if any(decoded[row * scanline_size] > 4 for row in range(height)): + raise EvidenceError("pack PNG contains an invalid scanline filter byte") + return width, height + + +def _run_cli(arguments: Sequence[str]) -> bytes: + environment = { + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": os.environ.get("PATH", ""), + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + "TZ": "UTC", + } + completed = subprocess.run( + [sys.executable, "-B", "-m", "git_dag_lab", *arguments], + cwd=ROOT, + env=environment, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + shell=False, + timeout=30, + ) + if completed.returncode != 0: + raise EvidenceError(f"CLI {' '.join(arguments)} failed") + if completed.stderr: + raise EvidenceError(f"CLI {' '.join(arguments)} wrote to stderr") + return completed.stdout + + +def _collect_cli() -> tuple[bytes, bytes, bytes, dict[str, Any]]: + first = ( + _run_cli(("pack-verify",)), + _run_cli(("pack-inspect", "--compact")), + _run_cli(("pack-inspect",)), + ) + second = ( + _run_cli(("pack-verify",)), + _run_cli(("pack-inspect", "--compact")), + _run_cli(("pack-inspect",)), + ) + if first != second: + raise EvidenceError("two fresh pack CLI runs produced different evidence") + verify_output, compact_output, pretty_output = first + try: + compact_document = json.loads(compact_output) + pretty_document = json.loads(pretty_output) + except json.JSONDecodeError as exc: + raise EvidenceError("pack inspection output is not JSON") from exc + if compact_document != pretty_document: + raise EvidenceError("compact and pretty pack documents differ") + if compact_output != _json_bytes(compact_document, pretty=False): + raise EvidenceError("compact pack inspection output is not canonical JSON") + if pretty_output != _json_bytes(pretty_document, pretty=True): + raise EvidenceError("pretty pack inspection output is not canonical JSON") + _validate_document(compact_document, verify_output) + return verify_output, compact_output, pretty_output, compact_document + + +def _validate_document(document: Mapping[str, Any], verify_output: bytes) -> None: + if set(document) != {"receipt", "report"}: + raise EvidenceError("pack evidence has unexpected top-level fields") + report = document.get("report") + receipt = document.get("receipt") + if not isinstance(report, dict) or not isinstance(receipt, dict): + raise EvidenceError("pack evidence sections are malformed") + if report.get("schema_version") != "git-pack-index-lab/v1": + raise EvidenceError("unexpected pack evidence schema") + receipt_sha = receipt.get("sha256") + if not isinstance(receipt_sha, str) or not SHA256_RE.fullmatch(receipt_sha): + raise EvidenceError("pack receipt is malformed") + canonical = json.dumps( + report, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + if _sha256(canonical) != receipt_sha: + raise EvidenceError("pack receipt does not bind the canonical report") + if f"receipt_sha256={receipt_sha}\n".encode("ascii") not in verify_output: + raise EvidenceError("pack transcript and receipt differ") + + pack = report.get("pack", {}) + index = report.get("index", {}) + checks = report.get("checks", {}) + scope = report.get("scope", {}) + objects = report.get("objects_in_pack_order", []) + fixture = report.get("fixture", {}) + if ( + pack.get("version") != 2 + or pack.get("object_count") != 3 + or pack.get("delta_count") != 0 + or not isinstance(pack.get("bytes"), int) + or pack["bytes"] <= 32 + or not SHA1_RE.fullmatch(pack.get("trailer_sha1", "")) + or not SHA256_RE.fullmatch(pack.get("sha256", "")) + ): + raise EvidenceError("pack summary is outside the reviewed fixture") + if ( + index.get("version") != 2 + or index.get("pack_sha1") != pack["trailer_sha1"] + or not isinstance(index.get("bytes"), int) + or index["bytes"] <= 1_064 + or not SHA1_RE.fullmatch(index.get("index_sha1", "")) + or not SHA256_RE.fullmatch(index.get("sha256", "")) + ): + raise EvidenceError("index summary is outside the reviewed fixture") + expected_checks = { + "all_fixture_objects_present", + "delta_entries_absent", + "index_checksum_verified", + "index_crc32_matches_pack", + "index_fanout_matches_sorted_oids", + "index_offsets_match_pack", + "pack_trailer_verified", + } + if set(checks) != expected_checks or not all(value is True for value in checks.values()): + raise EvidenceError("not every pack/index check passed") + if ( + fixture.get("object_count") != 3 + or len(fixture.get("objects", [])) != 3 + or len(objects) != 3 + ): + raise EvidenceError("pack fixture inventory is incomplete") + for row in objects: + if ( + row.get("object_type") != "blob" + or not SHA1_RE.fullmatch(row.get("oid", "")) + or not SHA256_RE.fullmatch(row.get("payload_sha256", "")) + or row.get("offset") != row.get("index_offset") + or row.get("crc32") != row.get("index_crc32") + ): + raise EvidenceError("pack object row is not cross-bound") + if ( + scope.get("delta_entries_supported") is not False + or scope.get("authentication_claim") is not False + or scope.get("git_pack_objects_executed") is not True + or scope.get("network_required") is not False + ): + raise EvidenceError("pack scope and non-claims are incomplete") + rendered = json.dumps(document, ensure_ascii=True, sort_keys=True) + forbidden = ("/home/", "github.com/", "@gmail.com", "AKIA", "ghp_", "github_pat_") + if any(marker.lower() in rendered.lower() for marker in forbidden): + raise EvidenceError("pack evidence contains host, remote, or credential material") + + +def _layout_svg(document: Mapping[str, Any]) -> bytes: + report = document["report"] + receipt = document["receipt"]["sha256"] + pack = report["pack"] + index = report["index"] + objects = report["objects_in_pack_order"] + total = pack["bytes"] + x0 = 55 + width = 1_330 + colors = ("#59e3c2", "#bd7cff", "#55a8ff") + parts = [ + _pack_svg_header( + "Actual Git pack v2 and index v2 byte layout", + "Byte offsets and section sizes are derived from the real production pack receipt.", + width=1440, + height=700, + receipt=receipt, + metadata_extra={ + "index_sha256": index["sha256"], + "pack_sha256": pack["sha256"], + }, + ), + ' REAL PACK V2 · ACTUAL OFFSETS · THREE NON-DELTA BLOBS', + f' {total} pack bytes, independently decoded', + ' ', + ] + header_width = max(58, round(12 / total * width)) + parts.append( + f' ' + ) + parts.append( + f' PACK' + ) + for number, (row, color) in enumerate(zip(objects, colors, strict=True), 1): + x = x0 + round(row["offset"] / total * width) + segment_width = max(105, round(row["packed_size"] / total * width)) + parts.extend( + ( + f' ', + f' 0{number} · {row["label"]}', + f' offset {row["offset"]}', + f' {row["packed_size"]} bytes', + ) + ) + trailer_x = x0 + round((total - 20) / total * width) + trailer_width = max(82, round(20 / total * width)) + parts.extend( + ( + f' ', + f' SHA-1', + f' 20 B', + ' INDEX V2 FIXED TABLES', + ) + ) + # Widths are legibility-scaled; every label retains the actual byte size. + sections = ( + ("header", 8, 90, "#243d62"), + ("fanout[256]", 1_024, 650, "#55a8ff"), + ("sorted OIDs", 60, 180, "#59e3c2"), + ("CRC32", 12, 130, "#bd7cff"), + ("offsets", 12, 130, "#ff8fa3"), + ("checksums", 40, 150, "#ffcf66"), + ) + index_x = 55 + for name, size, section_width, color in sections: + parts.extend( + ( + f' ', + f' {name}', + f' {size} B', + ) + ) + index_x += section_width + parts.extend( + ( + f' ', + f' Pack trailer SHA-1 · {_xml_text(pack["trailer_sha1"])}', + f' Index SHA-1 · {_xml_text(index["index_sha1"])} · binds the same pack checksum', + f' Receipt SHA-256 · {receipt}', + "\n", + ) + ) + return "\n".join(parts).encode("utf-8") + + +def _fanout_svg(document: Mapping[str, Any]) -> bytes: + report = document["report"] + receipt = document["receipt"]["sha256"] + objects = sorted(report["objects_in_pack_order"], key=lambda row: row["oid"]) + buckets = { + row["prefix"]: row for row in report["index"]["nonzero_fanout_buckets"] + } + parts = [ + _pack_svg_header( + "Actual index v2 fanout lookup", + "The three populated prefix buckets, sorted object IDs, and exact index ranges come from the verified index receipt.", + width=1440, + height=690, + receipt=receipt, + ), + ' 256-BUCKET CUMULATIVE FANOUT · THREE ACTUAL OBJECT IDS', + ' Prefix lookup narrows the sorted OID table.', + ] + for position, row in enumerate(objects): + y = 160 + (position * 145) + bucket = buckets[row["oid"][:2]] + parts.extend( + ( + f' ', + f' PREFIX {row["oid"][:2]}', + f' [{bucket["range_start"]}, {bucket["cumulative"]})', + f' ', + f' ', + f' {_xml_text(row["label"])} · blob · {row["size"]} bytes', + f' {row["oid"]}', + ) + ) + parts.extend( + ( + ' ', + f' Every displayed bucket and range is decoded from index bytes · receipt {receipt}', + "\n", + ) + ) + return "\n".join(parts).encode("utf-8") + + +def _integrity_svg(document: Mapping[str, Any]) -> bytes: + report = document["report"] + receipt = document["receipt"]["sha256"] + pack = report["pack"] + index = report["index"] + stages = ( + ("PACK HEADER", f"v{pack['version']} · {pack['object_count']} objects"), + ("ENTRY STREAMS", "bounded zlib · logical OIDs"), + ("INDEX ROWS", "CRC32 · exact offsets"), + ("PACK BINDING", index["pack_sha1"]), + ("INDEX CHECKSUM", index["index_sha1"]), + ) + parts = [ + _pack_svg_header( + "Independent pack/index integrity chain", + "The production verifier reconstructs each logical object, checks entry CRCs and offsets, then binds both file checksums.", + width=1440, + height=580, + receipt=receipt, + ), + ' GIT WRITES · PYTHON INDEPENDENTLY READS · NO MODELLED SUCCESS', + ' Five checks connect raw bytes to one receipt.', + ] + x_values = (35, 315, 595, 875, 1155) + for number, ((title, detail), x) in enumerate(zip(stages, x_values, strict=True), 1): + visible = detail if len(detail) <= 24 else detail[:20] + "…" + parts.extend( + ( + f' ', + f' 0{number}', + f' {title}', + f' {_xml_text(visible)}', + f' PASS', + ) + ) + if number < len(stages): + parts.append(f' ') + parts.extend( + ( + ' ', + f' 3/3 CRC32 and offsets cross-match · 0 deltas accepted · pack SHA-1 {pack["trailer_sha1"]}', + f' SHA-1 models Git object storage; authentication claim = false · receipt {receipt}', + "\n", + ) + ) + return "\n".join(parts).encode("utf-8") + + +def _cli_svg(document: Mapping[str, Any], verify_output: bytes) -> bytes: + receipt = document["receipt"]["sha256"] + transcript = verify_output.decode("utf-8").rstrip("\n") + before_receipt, receipt_digest = transcript.rsplit(" receipt_sha256=", 1) + before_pack, pack_digest = before_receipt.rsplit("pack_sha1=", 1) + parts = [ + _pack_svg_header( + "Real Git pack/index CLI receipt", + "Exact stdout from pack-verify; the command executed real git pack-objects and independently checked both files.", + width=1440, + height=500, + receipt=receipt, + ), + ' REAL CLI STDOUT · STDERR 0 BYTES · EXIT 0', + ' ', + ' ', + ' $ python3 -B -m git_dag_lab pack-verify', + f' {_xml_text(before_pack)}', + f' pack_sha1={pack_digest}', + f' receipt_sha256={receipt_digest}', + f' verify.txt SHA-256 · {_sha256(verify_output)}', + f' Report receipt SHA-256 · {receipt}', + "\n", + ] + return "\n".join(parts).encode("utf-8") + + +def _report_html(document: Mapping[str, Any], verify_output: bytes) -> bytes: + report = document["report"] + receipt = document["receipt"]["sha256"] + pack = report["pack"] + index = report["index"] + checks = report["checks"] + objects = report["objects_in_pack_order"] + transcript = html.escape(verify_output.decode("utf-8").rstrip("\n")) + object_rows = "".join( + f'''{html.escape(row["label"])}{row["oid"]}{row["size"]}{row["offset"]}{row["crc32"]}''' + for row in objects + ) + check_rows = "".join( + f"
  • ✓{html.escape(name.replace('_', ' '))}
    • " + for name, passed in checks.items() + if passed + ) + text = f''' + + + + + + Git pack/index evidence — verified offline report + + + +
    +

    Git pack/index lab · actual offline report

    +

    Pack v2.
    Index v2.

    +

    Real git pack-objects output is decoded independently: bounded zlib entries, logical object IDs, fanout ranges, CRC32 rows, offsets, and both file checksums.

    +
    PASS · report receipt {receipt}
    +
    +
    3packed objects
    +
    {pack["bytes"]}actual pack bytes
    +
    {index["bytes"]}actual index bytes
    +
    7/7checks passed
    +
    +
    +

    Objects in physical pack order

    {object_rows}
    fixtureOIDsizeoffsetCRC32
    +

    Independent gate

      {check_rows}
    +
    +

    Real CLI receipt

    $ python3 -B -m git_dag_lab pack-verify\n{transcript}
    Pack trailer
    {pack["trailer_sha1"]}

    Index checksum
    {index["index_sha1"]}

    Delta support: false · authentication claim: false · network required: false

    +
    Generated offline · no JavaScript · no external assetsevidence/git-pack-index-v1.json
    +
    + + +''' + return text.encode("utf-8") + + +def _capture_path_is_safe_if_present(path: Path) -> None: + target = ROOT / path + if target.exists() and (not target.is_file() or target.is_symlink()): + raise EvidenceError(f"unsafe pack capture path: {path.as_posix()}") + + +def _load_capture( + report_content: bytes, + receipt: str, + *, + allow_missing: bool, +) -> PackCapture | None: + for path in (SCREENSHOT_PATH, RENDERED_DOM_PATH, ATTESTATION_PATH): + _capture_path_is_safe_if_present(path) + if allow_missing: + return None + present = [ + (ROOT / path).is_file() + for path in (SCREENSHOT_PATH, RENDERED_DOM_PATH, ATTESTATION_PATH) + ] + if not all(present): + raise EvidenceError("pack browser capture is missing") + + screenshot = _safe_capture_bytes(SCREENSHOT_PATH, "pack screenshot") + rendered_dom = _safe_capture_bytes(RENDERED_DOM_PATH, "pack rendered DOM") + attestation = _safe_capture_bytes(ATTESTATION_PATH, "pack capture attestation") + if _parse_pack_png(screenshot) != PNG_DIMENSIONS: + raise EvidenceError("pack screenshot dimensions differ from the contract") + try: + dom_text = rendered_dom.decode("utf-8") + document = json.loads(attestation) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise EvidenceError("pack browser capture metadata is malformed") from exc + required = ( + f'data-pack-receipt="{receipt}"', + 'data-object-count="3"', + 'data-check-count="7"', + "Pack v2.", + "Index v2.", + "packed objects", + ) + if not all(marker in dom_text for marker in required): + raise EvidenceError("pack rendered DOM sentinels are incomplete") + if any(marker in dom_text for marker in ("/home/", "ERR_FILE", "github.com/")): + raise EvidenceError("pack rendered DOM contains a forbidden marker") + if set(document) != {"attestation", "receipt"}: + raise EvidenceError("pack capture attestation shape is invalid") + payload = document["attestation"] + attestation_receipt = document["receipt"].get("sha256") + canonical = json.dumps( + payload, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + if attestation_receipt != _sha256(canonical): + raise EvidenceError("pack capture attestation receipt is invalid") + try: + report_row = payload["input"]["report"] + dom_row = payload["outputs"]["rendered_dom"] + screenshot_row = payload["outputs"]["screenshot"] + script_row = payload["script"] + except (KeyError, TypeError) as exc: + raise EvidenceError("pack capture attestation rows are incomplete") from exc + script_path = Path("tools/capture_pack_report.sh") + script_content = _safe_capture_bytes(script_path, "pack capture script") + if ( + payload.get("schema_version") != "git-pack-browser-capture-attestation/v1" + or payload.get("browser", {}).get("binary_path") != BROWSER_PATH + or payload.get("browser", {}).get("sha256") != BROWSER_SHA256 + or payload.get("browser", {}).get("version") != BROWSER_VERSION + or payload.get("container", {}).get("image") != CONTAINER_IMAGE + or payload.get("container", {}).get("architecture") != "amd64" + or payload.get("isolation", {}).get("network") != "none" + or payload.get("isolation", {}).get("root_filesystem") != "read-only" + or payload.get("isolation", {}).get("demo_mount") != "read-only" + or payload.get("isolation", {}).get("capabilities") != "all-dropped" + or payload.get("isolation", {}).get("no_new_privileges") is not True + or payload.get("viewport") != { + "device_scale_factor": 1, + "height": PNG_DIMENSIONS[1], + "width": PNG_DIMENSIONS[0], + } + or report_row.get("path") != REPORT_PATH.as_posix() + or report_row.get("sha256") != _sha256(report_content) + or report_row.get("size") != len(report_content) + or report_row.get("report_receipt_sha256") != receipt + or dom_row.get("path") != RENDERED_DOM_PATH.as_posix() + or dom_row.get("sha256") != _sha256(rendered_dom) + or dom_row.get("size") != len(rendered_dom) + or screenshot_row.get("path") != SCREENSHOT_PATH.as_posix() + or screenshot_row.get("sha256") != _sha256(screenshot) + or screenshot_row.get("size") != len(screenshot) + or screenshot_row.get("width") != PNG_DIMENSIONS[0] + or screenshot_row.get("height") != PNG_DIMENSIONS[1] + or script_row.get("path") != script_path.as_posix() + or script_row.get("sha256") != _sha256(script_content) + or script_row.get("size") != len(script_content) + ): + raise EvidenceError("pack capture attestation does not bind current inputs") + return PackCapture( + screenshot=screenshot, + rendered_dom=rendered_dom, + attestation=attestation, + document=document, + ) + + +def build_artifacts(*, allow_missing_screenshot: bool) -> dict[Path, bytes]: + verify_output, compact_output, pretty_output, document = _collect_cli() + report_content = _report_html(document, verify_output) + receipt = document["receipt"]["sha256"] + capture = _load_capture( + report_content, + receipt, + allow_missing=allow_missing_screenshot, + ) + generated = { + EVIDENCE_PATH: compact_output, + VERIFY_PATH: verify_output, + INSPECT_PATH: pretty_output, + REPORT_PATH: report_content, + LAYOUT_PATH: _layout_svg(document), + FANOUT_PATH: _fanout_svg(document), + INTEGRITY_PATH: _integrity_svg(document), + CLI_PATH: _cli_svg(document, verify_output), + } + roles = { + EVIDENCE_PATH: "canonical compact pack/index CLI evidence", + VERIFY_PATH: "exact pack-verify stdout", + INSPECT_PATH: "exact pretty pack-inspect stdout", + REPORT_PATH: "dependency-free pack/index offline report", + LAYOUT_PATH: "actual pack and index byte layout", + FANOUT_PATH: "actual index fanout ranges and sorted object IDs", + INTEGRITY_PATH: "pack/index integrity chain derived from the receipt", + CLI_PATH: "visualized exact pack-verify transcript", + } + artifact_rows = [ + _artifact_row(path, generated[path], roles[path]) for path in sorted(generated) + ] + capture_manifest: dict[str, Any] = { + "attestation": ATTESTATION_PATH.as_posix(), + "browser_binary": BROWSER_PATH, + "browser_sha256": BROWSER_SHA256, + "browser_version": BROWSER_VERSION, + "container_image": CONTAINER_IMAGE, + "network": "none", + "status": "not-attested", + "viewport": {"height": PNG_DIMENSIONS[1], "width": PNG_DIMENSIONS[0]}, + } + if capture is not None: + artifact_rows.extend( + ( + _artifact_row( + SCREENSHOT_PATH, + capture.screenshot, + "attested pack/index offline report browser capture", + ), + _artifact_row( + RENDERED_DOM_PATH, + capture.rendered_dom, + "actual DOM emitted during the pack report capture", + ), + _artifact_row( + ATTESTATION_PATH, + capture.attestation, + "pack capture provenance and isolation attestation", + ), + ) + ) + capture_manifest.update( + { + "attestation_receipt_sha256": capture.document["receipt"]["sha256"], + "rendered_dom_sha256": _sha256(capture.rendered_dom), + "screenshot_sha256": _sha256(capture.screenshot), + "status": "attested", + } + ) + artifact_rows.sort(key=lambda row: row["path"]) + manifest = { + "artifacts": artifact_rows, + "capture": capture_manifest, + "commands": [ + { + "argv": ["python3", "-B", "-m", "git_dag_lab", "pack-verify"], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": VERIFY_PATH.as_posix(), + }, + { + "argv": [ + "python3", + "-B", + "-m", + "git_dag_lab", + "pack-inspect", + "--compact", + ], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": EVIDENCE_PATH.as_posix(), + }, + { + "argv": ["python3", "-B", "-m", "git_dag_lab", "pack-inspect"], + "exit_code": 0, + "fresh_runs": 2, + "stderr_bytes": 0, + "stdout": INSPECT_PATH.as_posix(), + }, + ], + "report_receipt_sha256": receipt, + "schema_version": "git-pack-index-evidence-manifest/v1", + "sources": [ + _source_row(Path("git_dag_lab/pack.py")), + _source_row(Path("git_dag_lab/cli.py")), + _source_row(Path("tools/generate_pack_evidence.py")), + _source_row(Path("tools/capture_pack_report.sh")), + ], + } + generated[MANIFEST_PATH] = _json_bytes(manifest, pretty=True) + _validate_generated(generated, capture, allow_missing_screenshot) + return generated + + +def _validate_generated( + generated: Mapping[Path, bytes], + capture: PackCapture | None, + allow_missing_screenshot: bool, +) -> None: + manifest = json.loads(generated[MANIFEST_PATH]) + rows = {row["path"]: row for row in manifest["artifacts"]} + for path, content in generated.items(): + if path == MANIFEST_PATH: + continue + row = rows.get(path.as_posix()) + if row is None or row["sha256"] != _sha256(content) or row["size"] != len(content): + raise EvidenceError(f"pack manifest does not bind {path.as_posix()}") + if capture is not None: + external = { + SCREENSHOT_PATH: capture.screenshot, + RENDERED_DOM_PATH: capture.rendered_dom, + ATTESTATION_PATH: capture.attestation, + } + for path, content in external.items(): + row = rows.get(path.as_posix()) + if row is None or row["sha256"] != _sha256(content) or row["size"] != len(content): + raise EvidenceError(f"pack manifest does not bind {path.as_posix()}") + if manifest["capture"].get("status") != "attested": + raise EvidenceError("pack manifest omits its attested capture") + elif not allow_missing_screenshot: + raise EvidenceError("pack manifest has no required browser capture") + + receipt = manifest["report_receipt_sha256"] + for path in (LAYOUT_PATH, FANOUT_PATH, INTEGRITY_PATH, CLI_PATH, REPORT_PATH): + text = generated[path].decode("utf-8") + if receipt not in text: + raise EvidenceError(f"{path.as_posix()} is not receipt-bound") + without_namespace = text.replace("http://www.w3.org/2000/svg", "") + if ( + "https://" in without_namespace + or "http://" in without_namespace + or "/home/" in without_namespace + ): + raise EvidenceError(f"{path.as_posix()} contains an external reference") + for path in (LAYOUT_PATH, FANOUT_PATH, INTEGRITY_PATH, CLI_PATH): + text = generated[path].decode("utf-8") + if " argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + action = parser.add_mutually_exclusive_group(required=True) + action.add_argument("--write", action="store_true", help="write generated evidence") + action.add_argument("--check", action="store_true", help="verify checked-in evidence") + parser.add_argument( + "--allow-missing-screenshot", + action="store_true", + help="permit the pre-capture package used by capture_pack_report.sh", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_parser().parse_args(argv) + try: + generated = build_artifacts( + allow_missing_screenshot=args.allow_missing_screenshot + ) + if args.write: + _write_artifacts(generated) + else: + _check_artifacts(generated) + except EvidenceError as exc: + print(f"pack evidence error: {exc}", file=sys.stderr) + return 1 + action = "wrote" if args.write else "verified" + screenshot = "optional" if args.allow_missing_screenshot else "required" + print( + f"PASS pack-evidence {action} artifacts={len(generated)} " + f"screenshot={screenshot}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())