From 0cdfbebc48439fc2ffbf2858af05dd34d59bfe04 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Sun, 9 Aug 2026 11:58:47 +0000 Subject: [PATCH 01/12] Add closed-subset Git pack and index verifier --- git_dag_lab/__init__.py | 5 + git_dag_lab/cli.py | 22 +- git_dag_lab/lab.py | 1 + git_dag_lab/pack.py | 612 ++++++++++++++++++++++++++++++++++++++++ tests/test_pack_lab.py | 396 ++++++++++++++++++++++++++ 5 files changed, 1034 insertions(+), 2 deletions(-) create mode 100644 git_dag_lab/pack.py create mode 100644 tests/test_pack_lab.py diff --git a/git_dag_lab/__init__.py b/git_dag_lab/__init__.py index 4d5c901..35ff0d1 100644 --- a/git_dag_lab/__init__.py +++ b/git_dag_lab/__init__.py @@ -1,14 +1,19 @@ """Deterministic experiments with real Git objects.""" from .lab import LabError, LabReport, git_object_oid, parse_commit, parse_tree, run_lab +from .pack import PackReport, parse_index, parse_pack, run_pack_lab __all__ = [ "LabError", "LabReport", + "PackReport", "git_object_oid", + "parse_index", + "parse_pack", "parse_commit", "parse_tree", "run_lab", + "run_pack_lab", ] __version__ = "0.1.0" diff --git a/git_dag_lab/cli.py b/git_dag_lab/cli.py index 4528062..95185c3 100644 --- a/git_dag_lab/cli.py +++ b/git_dag_lab/cli.py @@ -9,6 +9,7 @@ from typing import TextIO from .lab import LabError, run_lab +from .pack import run_pack_lab def build_parser() -> argparse.ArgumentParser: @@ -27,6 +28,20 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="emit canonical JSON on one line instead of indented JSON", ) + + subparsers.add_parser( + "pack-verify", + help="build and independently verify a real Git pack v2/index v2 pair", + ) + pack_inspect_parser = subparsers.add_parser( + "pack-inspect", + help="print the complete machine-readable pack/index evidence document", + ) + pack_inspect_parser.add_argument( + "--compact", + action="store_true", + help="emit canonical JSON on one line instead of indented JSON", + ) return parser @@ -44,12 +59,15 @@ def main( args = build_parser().parse_args(argv) try: - report = run_lab(root if root is not None else Path.cwd()) + if args.command.startswith("pack-"): + report = run_pack_lab(root if root is not None else Path.cwd()) + else: + report = run_lab(root if root is not None else Path.cwd()) except LabError as exc: errors.write(f"ERROR git-dag-lab: {exc}\n") return 1 - if args.command == "verify": + if args.command in {"verify", "pack-verify"}: output.write(report.receipt_line + "\n") else: output.write(report.to_json(pretty=not args.compact) + "\n") diff --git a/git_dag_lab/lab.py b/git_dag_lab/lab.py index 21a79b5..3e70cd6 100644 --- a/git_dag_lab/lab.py +++ b/git_dag_lab/lab.py @@ -54,6 +54,7 @@ "hash-object", "merge-base", "mktree", + "pack-objects", "rev-list", "symbolic-ref", "update-ref", diff --git a/git_dag_lab/pack.py b/git_dag_lab/pack.py new file mode 100644 index 0000000..0da23da --- /dev/null +++ b/git_dag_lab/pack.py @@ -0,0 +1,612 @@ +"""Build and independently verify a bounded, deterministic Git pack/index pair. + +The fixture uses real ``git pack-objects`` output but validates the pack v2 and +index v2 bytes with Python's standard library. Delta entries are disabled for +this first closed subset and rejected by the parser. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import binascii +import hashlib +import json +import os +from pathlib import Path +import stat +import tempfile +from typing import Any +import zlib + +from .lab import ( + LabError, + VerificationError, + _GitRunner, + _canonical_json, + _deep_freeze, + _find_git, + _mutable_copy, + _new_workspace, + _store_blob, + _validate_root, + git_object_oid, +) + +PACK_SCHEMA_VERSION = "git-pack-index-lab/v1" +PACK_VERSION = 2 +INDEX_VERSION = 2 +PACK_MAGIC = b"PACK" +INDEX_MAGIC = b"\xfftOc" +MAX_PACK_BYTES = 1_048_576 +MAX_PACK_OBJECTS = 64 +MAX_OBJECT_BYTES = 262_144 + +PACK_BLOBS = ( + ("binary-header", bytes(range(32))), + ("content-addressing", b"content-addressed systems\n"), + ( + "index-fanout", + b"fanout tables map object-id prefixes to sorted index ranges\n", + ), +) +_TYPE_BY_CODE = {1: "commit", 2: "tree", 3: "blob", 4: "tag"} + + +@dataclass(frozen=True, slots=True) +class PackEntry: + """One independently decoded non-delta pack entry.""" + + crc32: int + object_type: str + offset: int + oid: str + packed_size: int + payload_sha256: str + size: int + + def as_dict(self) -> dict[str, object]: + return { + "crc32": f"{self.crc32:08x}", + "object_type": self.object_type, + "offset": self.offset, + "oid": self.oid, + "packed_size": self.packed_size, + "payload_sha256": self.payload_sha256, + "size": self.size, + } + + +@dataclass(frozen=True, slots=True) +class ParsedPack: + """Verified pack header, entries, and trailer.""" + + entries: tuple[PackEntry, ...] + trailer_sha1: str + version: int + + +@dataclass(frozen=True, slots=True) +class IndexEntry: + """One verified index v2 row.""" + + crc32: int + offset: int + oid: str + + def as_dict(self) -> dict[str, object]: + return { + "crc32": f"{self.crc32:08x}", + "offset": self.offset, + "oid": self.oid, + } + + +@dataclass(frozen=True, slots=True) +class ParsedIndex: + """Verified index v2 fanout, rows, and checksums.""" + + entries: tuple[IndexEntry, ...] + fanout: tuple[int, ...] + index_sha1: str + pack_sha1: str + version: int + + +@dataclass(frozen=True, slots=True) +class PackReport: + """Canonical evidence document for one real pack/index build.""" + + payload: Mapping[str, Any] + receipt_sha256: str + + @property + def document(self) -> dict[str, Any]: + return { + "report": _mutable_copy(self.payload), + "receipt": { + "algorithm": "sha256", + "canonicalization": "UTF-8 JSON; sorted keys; compact separators", + "sha256": self.receipt_sha256, + }, + } + + @property + def receipt_line(self) -> str: + pack = self.payload["pack"] + index = self.payload["index"] + return ( + f"PASS {PACK_SCHEMA_VERSION} " + f"objects={pack['object_count']} " + f"pack_version={pack['version']} " + f"index_version={index['version']} " + "deltas=0 " + f"pack_sha1={pack['trailer_sha1']} " + f"receipt_sha256={self.receipt_sha256}" + ) + + def to_json(self, *, pretty: bool = False) -> str: + if pretty: + return json.dumps( + self.document, + ensure_ascii=True, + indent=2, + sort_keys=True, + ) + return _canonical_json(self.document).decode("utf-8") + + +def _uint32(content: bytes, offset: int, *, label: str) -> int: + end = offset + 4 + if offset < 0 or end > len(content): + raise VerificationError(f"{label} is truncated") + return int.from_bytes(content[offset:end], "big") + + +def _uint64(content: bytes, offset: int, *, label: str) -> int: + end = offset + 8 + if offset < 0 or end > len(content): + raise VerificationError(f"{label} is truncated") + return int.from_bytes(content[offset:end], "big") + + +def _read_regular_file(path: Path, *, label: str) -> bytes: + """Read one bounded, single-link regular file without following a symlink.""" + + try: + before = path.lstat() + except OSError as exc: + raise VerificationError(f"{label} is unavailable") from exc + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or before.st_size < 1 + or before.st_size > MAX_PACK_BYTES + ): + raise VerificationError(f"{label} is not a bounded single-link file") + + flags = os.O_RDONLY + flags |= getattr(os, "O_CLOEXEC", 0) + flags |= getattr(os, "O_NOFOLLOW", 0) + try: + descriptor = os.open(path, flags) + except OSError as exc: + raise VerificationError(f"{label} could not be opened safely") from exc + try: + observed = os.fstat(descriptor) + if ( + not stat.S_ISREG(observed.st_mode) + or observed.st_nlink != 1 + or observed.st_dev != before.st_dev + or observed.st_ino != before.st_ino + or observed.st_size != before.st_size + ): + raise VerificationError(f"{label} changed before it was read") + chunks: list[bytes] = [] + remaining = observed.st_size + while remaining: + chunk = os.read(descriptor, min(65_536, remaining)) + if not chunk: + raise VerificationError(f"{label} ended before its declared size") + chunks.append(chunk) + remaining -= len(chunk) + if os.read(descriptor, 1): + raise VerificationError(f"{label} grew while it was read") + finally: + os.close(descriptor) + return b"".join(chunks) + + +def _decode_entry_header( + content: bytes, + offset: int, + *, + end: int, +) -> tuple[str, int, int]: + if offset >= end: + raise VerificationError("pack entry header is truncated") + first = content[offset] + offset += 1 + type_code = (first >> 4) & 0x07 + if type_code not in _TYPE_BY_CODE: + if type_code in {6, 7}: + raise VerificationError("delta pack entries are outside the reviewed subset") + raise VerificationError("pack entry type is invalid") + + size = first & 0x0F + shift = 4 + current = first + while current & 0x80: + if offset >= end or shift > 60: + raise VerificationError("pack entry size header is malformed") + current = content[offset] + offset += 1 + size |= (current & 0x7F) << shift + shift += 7 + if size > MAX_OBJECT_BYTES: + raise VerificationError("pack entry expands beyond the reviewed bound") + return _TYPE_BY_CODE[type_code], size, offset + + +def parse_pack(content: bytes) -> ParsedPack: + """Parse and verify one bounded SHA-1 pack v2 without invoking Git.""" + + if ( + type(content) is not bytes + or len(content) < 32 + or len(content) > MAX_PACK_BYTES + ): + raise VerificationError("pack bytes are outside the reviewed bound") + if content[:4] != PACK_MAGIC: + raise VerificationError("pack signature is invalid") + version = _uint32(content, 4, label="pack version") + if version != PACK_VERSION: + raise VerificationError("pack version is outside the reviewed subset") + count = _uint32(content, 8, label="pack object count") + if count < 1 or count > MAX_PACK_OBJECTS: + raise VerificationError("pack object count is outside the reviewed bound") + + pack_end = len(content) - 20 + trailer = content[pack_end:] + if hashlib.sha1(content[:pack_end], usedforsecurity=False).digest() != trailer: + raise VerificationError("pack trailer does not match the preceding bytes") + + entries: list[PackEntry] = [] + offset = 12 + seen: set[str] = set() + for _ in range(count): + entry_start = offset + object_type, declared_size, payload_start = _decode_entry_header( + content, + offset, + end=pack_end, + ) + inflater = zlib.decompressobj() + try: + payload = inflater.decompress( + content[payload_start:pack_end], + MAX_OBJECT_BYTES + 1, + ) + except zlib.error as exc: + raise VerificationError("pack entry zlib stream is invalid") from exc + if len(payload) > MAX_OBJECT_BYTES: + raise VerificationError("pack entry expands beyond the reviewed bound") + if not inflater.eof or inflater.unconsumed_tail: + raise VerificationError("pack entry zlib stream is incomplete or oversized") + consumed = pack_end - payload_start - len(inflater.unused_data) + if consumed < 1: + raise VerificationError("pack entry has an empty zlib stream") + offset = payload_start + consumed + if offset > pack_end or len(payload) != declared_size: + raise VerificationError("pack entry size does not match its payload") + + oid = git_object_oid(object_type, payload) + if oid in seen: + raise VerificationError("pack contains a duplicate logical object") + seen.add(oid) + packed = content[entry_start:offset] + entries.append( + PackEntry( + crc32=binascii.crc32(packed) & 0xFFFFFFFF, + object_type=object_type, + offset=entry_start, + oid=oid, + packed_size=len(packed), + payload_sha256=hashlib.sha256(payload).hexdigest(), + size=len(payload), + ) + ) + if offset != pack_end: + raise VerificationError("pack has trailing bytes outside its declared entries") + return ParsedPack( + entries=tuple(entries), + trailer_sha1=trailer.hex(), + version=version, + ) + + +def _expected_fanout(oids: tuple[str, ...]) -> tuple[int, ...]: + counts = [0] * 256 + for oid in oids: + counts[int(oid[:2], 16)] += 1 + total = 0 + fanout: list[int] = [] + for count in counts: + total += count + fanout.append(total) + return tuple(fanout) + + +def parse_index(content: bytes) -> ParsedIndex: + """Parse and verify one bounded Git index v2 without invoking Git.""" + + minimum = 8 + (256 * 4) + 40 + if ( + type(content) is not bytes + or len(content) < minimum + or len(content) > MAX_PACK_BYTES + ): + raise VerificationError("index bytes are outside the reviewed bound") + if content[:4] != INDEX_MAGIC: + raise VerificationError("index signature is invalid") + version = _uint32(content, 4, label="index version") + if version != INDEX_VERSION: + raise VerificationError("index version is outside the reviewed subset") + + fanout = tuple( + _uint32(content, 8 + (bucket * 4), label="index fanout") + for bucket in range(256) + ) + if any(left > right for left, right in zip(fanout, fanout[1:])): + raise VerificationError("index fanout table is not cumulative") + count = fanout[-1] + if count < 1 or count > MAX_PACK_OBJECTS: + raise VerificationError("index object count is outside the reviewed bound") + + oid_start = 8 + (256 * 4) + crc_start = oid_start + (count * 20) + offset_start = crc_start + (count * 4) + large_start = offset_start + (count * 4) + fixed_end = large_start + 40 + if fixed_end > len(content): + raise VerificationError("index tables are truncated") + + oids = tuple( + content[oid_start + (row * 20) : oid_start + ((row + 1) * 20)].hex() + for row in range(count) + ) + if tuple(sorted(oids)) != oids or len(set(oids)) != count: + raise VerificationError("index object IDs are not unique and sorted") + if fanout != _expected_fanout(oids): + raise VerificationError("index fanout does not match its object IDs") + + crc_values = tuple( + _uint32(content, crc_start + (row * 4), label="index CRC table") + for row in range(count) + ) + offset_words = tuple( + _uint32(content, offset_start + (row * 4), label="index offset table") + for row in range(count) + ) + large_indexes = [word & 0x7FFFFFFF for word in offset_words if word & 0x80000000] + if sorted(large_indexes) != list(range(len(large_indexes))): + raise VerificationError("index large-offset references are not canonical") + checksum_start = large_start + (len(large_indexes) * 8) + if checksum_start + 40 != len(content): + raise VerificationError("index has an invalid table length") + + large_offsets = tuple( + _uint64(content, large_start + (row * 8), label="index large-offset table") + for row in range(len(large_indexes)) + ) + offsets: list[int] = [] + for word in offset_words: + if word & 0x80000000: + offsets.append(large_offsets[word & 0x7FFFFFFF]) + else: + offsets.append(word) + + pack_sha1 = content[checksum_start : checksum_start + 20].hex() + index_sha1 = content[checksum_start + 20 :].hex() + expected_index_sha1 = hashlib.sha1( + content[: checksum_start + 20], + usedforsecurity=False, + ).hexdigest() + if index_sha1 != expected_index_sha1: + raise VerificationError("index checksum does not match the preceding bytes") + + entries = tuple( + IndexEntry(crc32=crc_values[row], offset=offsets[row], oid=oids[row]) + for row in range(count) + ) + return ParsedIndex( + entries=entries, + fanout=fanout, + index_sha1=index_sha1, + pack_sha1=pack_sha1, + version=version, + ) + + +def _cross_check(pack: ParsedPack, index: ParsedIndex) -> None: + if index.pack_sha1 != pack.trailer_sha1: + raise VerificationError("index does not bind the verified pack checksum") + pack_entries = {entry.oid: entry for entry in pack.entries} + if len(pack_entries) != len(index.entries): + raise VerificationError("pack and index object counts differ") + for indexed in index.entries: + packed = pack_entries.get(indexed.oid) + if packed is None: + raise VerificationError("index references an object absent from the pack") + if indexed.offset != packed.offset or indexed.crc32 != packed.crc32: + raise VerificationError("index offset or CRC does not match the pack entry") + + +def _build_and_verify_pack(runner: _GitRunner, private: Path) -> PackReport: + runner.initialize() + expected_payloads: dict[str, bytes] = {} + labels_by_oid: dict[str, str] = {} + for label, payload in PACK_BLOBS: + oid = _store_blob(runner, payload) + expected_payloads[oid] = payload + labels_by_oid[oid] = label + if len(expected_payloads) != len(PACK_BLOBS): + raise VerificationError("fixed pack fixture contains duplicate objects") + + ordered_oids = tuple(sorted(expected_payloads)) + prefix = private / "fixture" + result = runner.run( + "pack-objects", + "--window=0", + "--depth=0", + "--no-reuse-delta", + "--no-reuse-object", + os.fspath(prefix), + stdin=("".join(f"{oid}\n" for oid in ordered_oids)).encode("ascii"), + ) + if result.stderr: + raise VerificationError("git pack-objects produced unexpected diagnostics") + try: + pack_name = result.stdout.decode("ascii").strip() + except UnicodeDecodeError as exc: + raise VerificationError("git pack-objects returned a non-ASCII checksum") from exc + if ( + len(pack_name) != 40 + or any(character not in "0123456789abcdef" for character in pack_name) + ): + raise VerificationError("git pack-objects returned an invalid checksum") + + pack_path = Path(f"{prefix}-{pack_name}.pack") + index_path = Path(f"{prefix}-{pack_name}.idx") + pack_bytes = _read_regular_file(pack_path, label="generated pack") + index_bytes = _read_regular_file(index_path, label="generated index") + parsed_pack = parse_pack(pack_bytes) + parsed_index = parse_index(index_bytes) + _cross_check(parsed_pack, parsed_index) + if parsed_pack.trailer_sha1 != pack_name: + raise VerificationError("git pack name does not match the verified trailer") + if {entry.oid for entry in parsed_pack.entries} != set(expected_payloads): + raise VerificationError("generated pack object inventory is not exact") + for entry in parsed_pack.entries: + payload = expected_payloads[entry.oid] + if ( + entry.object_type != "blob" + or entry.size != len(payload) + or entry.payload_sha256 != hashlib.sha256(payload).hexdigest() + ): + raise VerificationError("generated pack object differs from the fixture") + + index_rows = {entry.oid: entry for entry in parsed_index.entries} + objects = [] + for entry in parsed_pack.entries: + indexed = index_rows[entry.oid] + objects.append( + { + **entry.as_dict(), + "index_crc32": f"{indexed.crc32:08x}", + "index_offset": indexed.offset, + "label": labels_by_oid[entry.oid], + } + ) + + nonzero_buckets = [ + { + "cumulative": parsed_index.fanout[bucket], + "prefix": f"{bucket:02x}", + "range_start": 0 if bucket == 0 else parsed_index.fanout[bucket - 1], + } + for bucket in range(256) + if ( + parsed_index.fanout[bucket] + != (0 if bucket == 0 else parsed_index.fanout[bucket - 1]) + ) + ] + payload: dict[str, object] = { + "checks": { + "all_fixture_objects_present": True, + "delta_entries_absent": True, + "index_checksum_verified": True, + "index_crc32_matches_pack": True, + "index_fanout_matches_sorted_oids": True, + "index_offsets_match_pack": True, + "pack_trailer_verified": True, + }, + "command_trace": list(runner.trace), + "fixture": { + "object_count": len(PACK_BLOBS), + "objects": [ + { + "label": label, + "oid": git_object_oid("blob", body), + "payload_sha256": hashlib.sha256(body).hexdigest(), + "size": len(body), + } + for label, body in PACK_BLOBS + ], + }, + "index": { + "bytes": len(index_bytes), + "index_sha1": parsed_index.index_sha1, + "nonzero_fanout_buckets": nonzero_buckets, + "pack_sha1": parsed_index.pack_sha1, + "sha256": hashlib.sha256(index_bytes).hexdigest(), + "version": parsed_index.version, + }, + "object_format": "sha1", + "objects_in_pack_order": objects, + "pack": { + "bytes": len(pack_bytes), + "delta_count": 0, + "object_count": len(parsed_pack.entries), + "sha256": hashlib.sha256(pack_bytes).hexdigest(), + "trailer_sha1": parsed_pack.trailer_sha1, + "version": parsed_pack.version, + }, + "scope": { + "authentication_claim": False, + "delta_entries_supported": False, + "fixture_kind": "three deterministic synthetic blobs", + "git_pack_objects_executed": True, + "network_required": False, + }, + "schema_version": PACK_SCHEMA_VERSION, + } + receipt = hashlib.sha256(_canonical_json(payload)).hexdigest() + return PackReport(payload=_deep_freeze(payload), receipt_sha256=receipt) + + +def run_pack_lab(root: Path | str = ".") -> PackReport: + """Build and verify a real pack/index pair below ``root``, then remove it.""" + + try: + validated_root = _validate_root(Path(root)) + git = _find_git() + with tempfile.TemporaryDirectory( + prefix=".git-pack-index-lab-", + dir=validated_root, + ) as private_name: + workspace = _new_workspace(validated_root, Path(private_name)) + runner = _GitRunner(git, workspace) + return _build_and_verify_pack(runner, workspace.private) + except LabError: + raise + except (OSError, TypeError, ValueError) as exc: + raise LabError("isolated pack lab setup failed") from exc + + +__all__ = [ + "INDEX_VERSION", + "MAX_OBJECT_BYTES", + "MAX_PACK_BYTES", + "MAX_PACK_OBJECTS", + "PACK_SCHEMA_VERSION", + "PACK_VERSION", + "IndexEntry", + "PackEntry", + "PackReport", + "ParsedIndex", + "ParsedPack", + "parse_index", + "parse_pack", + "run_pack_lab", +] diff --git a/tests/test_pack_lab.py b/tests/test_pack_lab.py new file mode 100644 index 0000000..8a4c843 --- /dev/null +++ b/tests/test_pack_lab.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import binascii +import hashlib +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +import zlib +from unittest import mock + +from git_dag_lab import pack as pack_module +from git_dag_lab.cli import main +from git_dag_lab.lab import VerificationError, git_object_oid +from git_dag_lab.pack import ( + INDEX_MAGIC, + INDEX_VERSION, + MAX_PACK_BYTES, + PACK_MAGIC, + PACK_SCHEMA_VERSION, + PACK_VERSION, + parse_index, + parse_pack, + run_pack_lab, +) + + +def _entry_header(type_code: int, size: int) -> bytes: + first = (type_code << 4) | (size & 0x0F) + size >>= 4 + output = bytearray((first,)) + if size: + output[0] |= 0x80 + while size: + current = size & 0x7F + size >>= 7 + if size: + current |= 0x80 + output.append(current) + return bytes(output) + + +def _resign_pack(prefix: bytes) -> bytes: + return prefix + hashlib.sha1(prefix, usedforsecurity=False).digest() + + +def _build_pack(payloads: tuple[bytes, ...]) -> tuple[bytes, list[dict[str, int | str]]]: + body = bytearray(PACK_MAGIC) + body.extend(PACK_VERSION.to_bytes(4, "big")) + body.extend(len(payloads).to_bytes(4, "big")) + rows: list[dict[str, int | str]] = [] + for payload in payloads: + offset = len(body) + packed = _entry_header(3, len(payload)) + zlib.compress(payload, level=9) + body.extend(packed) + rows.append( + { + "crc32": binascii.crc32(packed) & 0xFFFFFFFF, + "offset": offset, + "oid": git_object_oid("blob", payload), + } + ) + return _resign_pack(bytes(body)), rows + + +def _resign_index(prefix: bytes) -> bytes: + return prefix + hashlib.sha1(prefix, usedforsecurity=False).digest() + + +def _build_index( + rows: list[dict[str, int | str]], + pack_sha1: str, +) -> bytes: + ordered = sorted(rows, key=lambda row: str(row["oid"])) + counts = [0] * 256 + for row in ordered: + counts[int(str(row["oid"])[:2], 16)] += 1 + fanout: list[int] = [] + total = 0 + for count in counts: + total += count + fanout.append(total) + + body = bytearray(INDEX_MAGIC) + body.extend(INDEX_VERSION.to_bytes(4, "big")) + for value in fanout: + body.extend(value.to_bytes(4, "big")) + for row in ordered: + body.extend(bytes.fromhex(str(row["oid"]))) + for row in ordered: + body.extend(int(row["crc32"]).to_bytes(4, "big")) + for row in ordered: + body.extend(int(row["offset"]).to_bytes(4, "big")) + body.extend(bytes.fromhex(pack_sha1)) + return _resign_index(bytes(body)) + + +class PackParserTests(unittest.TestCase): + def setUp(self) -> None: + self.payloads = (b"fixture\n", b"\x00binary\xff\n") + self.pack_bytes, self.rows = _build_pack(self.payloads) + + def test_decodes_real_envelopes_and_verifies_trailer(self) -> None: + parsed = parse_pack(self.pack_bytes) + self.assertEqual(parsed.version, PACK_VERSION) + self.assertEqual(len(parsed.entries), 2) + self.assertEqual( + {entry.oid for entry in parsed.entries}, + {git_object_oid("blob", payload) for payload in self.payloads}, + ) + self.assertEqual( + parsed.trailer_sha1, + hashlib.sha1(self.pack_bytes[:-20], usedforsecurity=False).hexdigest(), + ) + for entry, payload in zip(parsed.entries, self.payloads, strict=True): + self.assertEqual(entry.object_type, "blob") + self.assertEqual(entry.size, len(payload)) + self.assertEqual(entry.payload_sha256, hashlib.sha256(payload).hexdigest()) + self.assertGreater(entry.packed_size, 1) + + def test_rejects_non_bytes_and_outer_boundary_drift(self) -> None: + cases = ( + bytearray(self.pack_bytes), + b"", + self.pack_bytes[:31], + b"x" * (MAX_PACK_BYTES + 1), + ) + for content in cases: + with self.subTest(size=len(content)), self.assertRaises(VerificationError): + parse_pack(content) # type: ignore[arg-type] + + def test_rejects_signature_version_and_count_drift(self) -> None: + bad_signature = b"FAIL" + self.pack_bytes[4:] + bad_version = bytearray(self.pack_bytes) + bad_version[4:8] = (3).to_bytes(4, "big") + bad_count = bytearray(self.pack_bytes) + bad_count[8:12] = (0).to_bytes(4, "big") + for content in ( + bad_signature, + _resign_pack(bytes(bad_version[:-20])), + _resign_pack(bytes(bad_count[:-20])), + ): + with self.assertRaises(VerificationError): + parse_pack(content) + + def test_rejects_delta_type_size_zlib_and_trailer_mutations(self) -> None: + delta = bytearray(self.pack_bytes[:-20]) + delta[12] = (delta[12] & 0x8F) | 0x60 + + wrong_size = bytearray(self.pack_bytes[:-20]) + wrong_size[12] = (wrong_size[12] & 0xF0) | ((len(self.payloads[0]) + 1) & 0x0F) + + corrupt_zlib = bytearray(self.pack_bytes[:-20]) + corrupt_zlib[14] ^= 0xFF + + bad_trailer = bytearray(self.pack_bytes) + bad_trailer[-1] ^= 0x01 + + for content in ( + _resign_pack(bytes(delta)), + _resign_pack(bytes(wrong_size)), + _resign_pack(bytes(corrupt_zlib)), + bytes(bad_trailer), + ): + with self.assertRaises(VerificationError): + parse_pack(content) + + def test_rejects_declared_extra_object_and_trailing_body_bytes(self) -> None: + extra_object = bytearray(self.pack_bytes[:-20]) + extra_object[8:12] = (3).to_bytes(4, "big") + trailing = self.pack_bytes[:-20] + b"\x00" + for content in ( + _resign_pack(bytes(extra_object)), + _resign_pack(trailing), + ): + with self.assertRaises(VerificationError): + parse_pack(content) + + +class IndexParserTests(unittest.TestCase): + def setUp(self) -> None: + self.pack_bytes, self.rows = _build_pack((b"alpha\n", b"omega\n")) + self.pack = parse_pack(self.pack_bytes) + self.index_bytes = _build_index(self.rows, self.pack.trailer_sha1) + + def test_decodes_fanout_rows_and_both_checksums(self) -> None: + parsed = parse_index(self.index_bytes) + self.assertEqual(parsed.version, INDEX_VERSION) + self.assertEqual(parsed.pack_sha1, self.pack.trailer_sha1) + self.assertEqual(parsed.fanout[-1], len(self.rows)) + self.assertEqual( + tuple(entry.oid for entry in parsed.entries), + tuple(sorted(str(row["oid"]) for row in self.rows)), + ) + self.assertEqual( + parsed.index_sha1, + hashlib.sha1(self.index_bytes[:-20], usedforsecurity=False).hexdigest(), + ) + pack_module._cross_check(self.pack, parsed) + + def test_rejects_signature_version_checksum_and_length_drift(self) -> None: + bad_version = bytearray(self.index_bytes) + bad_version[4:8] = (3).to_bytes(4, "big") + bad_checksum = bytearray(self.index_bytes) + bad_checksum[-1] ^= 0x01 + trailing = self.index_bytes[:-20] + b"\x00" + for content in ( + b"FAIL" + self.index_bytes[4:], + _resign_index(bytes(bad_version[:-20])), + bytes(bad_checksum), + _resign_index(trailing), + ): + with self.assertRaises(VerificationError): + parse_index(content) + + def test_rejects_fanout_oid_and_large_offset_drift(self) -> None: + oid_start = 8 + (256 * 4) + wrong_oid = bytearray(self.index_bytes[:-20]) + wrong_oid[oid_start] ^= 0x01 + + count = len(self.rows) + offset_start = oid_start + (count * 20) + (count * 4) + bad_large_offset = bytearray(self.index_bytes[:-20]) + bad_large_offset[offset_start : offset_start + 4] = ( + 0x80000001 + ).to_bytes(4, "big") + + for content in ( + _resign_index(bytes(wrong_oid)), + _resign_index(bytes(bad_large_offset)), + ): + with self.assertRaises(VerificationError): + parse_index(content) + + def test_cross_check_rejects_crc_offset_inventory_and_pack_binding_drift(self) -> None: + parsed = parse_index(self.index_bytes) + + changed_crc = bytearray(self.index_bytes[:-20]) + count = len(self.rows) + oid_start = 8 + (256 * 4) + crc_start = oid_start + (count * 20) + changed_crc[crc_start + 3] ^= 0x01 + + changed_offset = bytearray(self.index_bytes[:-20]) + offset_start = crc_start + (count * 4) + changed_offset[offset_start + 3] ^= 0x01 + + changed_pack = bytearray(self.index_bytes[:-20]) + changed_pack[-1] ^= 0x01 + + for content in ( + _resign_index(bytes(changed_crc)), + _resign_index(bytes(changed_offset)), + _resign_index(bytes(changed_pack)), + ): + with self.assertRaises(VerificationError): + pack_module._cross_check(self.pack, parse_index(content)) + self.assertEqual(len(parsed.entries), 2) + + +class PackRuntimeTests(unittest.TestCase): + def test_real_pack_and_index_are_deterministic_and_leave_no_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + first = run_pack_lab(root) + self.assertEqual(list(root.iterdir()), []) + second = run_pack_lab(root) + self.assertEqual(list(root.iterdir()), []) + + self.assertEqual(first.to_json(), second.to_json()) + report = first.document["report"] + self.assertEqual(report["schema_version"], PACK_SCHEMA_VERSION) + self.assertEqual(report["pack"]["version"], PACK_VERSION) + self.assertEqual(report["index"]["version"], INDEX_VERSION) + self.assertEqual(report["pack"]["object_count"], 3) + self.assertEqual(report["pack"]["delta_count"], 0) + self.assertEqual( + report["command_trace"], + ["init", "hash-object", "hash-object", "hash-object", "pack-objects"], + ) + self.assertTrue(all(report["checks"].values())) + self.assertFalse(report["scope"]["delta_entries_supported"]) + self.assertFalse(report["scope"]["authentication_claim"]) + + def test_receipt_binds_the_complete_payload(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + result = run_pack_lab(Path(temporary)) + document = result.document + canonical = json.dumps( + document["report"], + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + self.assertEqual( + document["receipt"]["sha256"], + hashlib.sha256(canonical).hexdigest(), + ) + self.assertEqual(json.loads(result.to_json()), document) + + def test_pack_cli_exposes_receipt_and_canonical_document(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + stdout = io.StringIO() + stderr = io.StringIO() + self.assertEqual( + main( + ["pack-verify"], + root=root, + stdout=stdout, + stderr=stderr, + ), + 0, + ) + self.assertEqual(stderr.getvalue(), "") + self.assertRegex( + stdout.getvalue(), + rf"^PASS {PACK_SCHEMA_VERSION} objects=3 .* receipt_sha256=[0-9a-f]{{64}}\n$", + ) + + inspect_output = io.StringIO() + self.assertEqual( + main( + ["pack-inspect", "--compact"], + root=root, + stdout=inspect_output, + stderr=io.StringIO(), + ), + 0, + ) + inspected = json.loads(inspect_output.getvalue()) + self.assertEqual(inspected["report"]["schema_version"], PACK_SCHEMA_VERSION) + + def test_pack_cli_returns_sanitized_lab_errors(self) -> None: + stdout = io.StringIO() + stderr = io.StringIO() + with mock.patch( + "git_dag_lab.cli.run_pack_lab", + side_effect=VerificationError("reviewed failure"), + ): + self.assertEqual( + main( + ["pack-verify"], + root=Path.cwd(), + stdout=stdout, + stderr=stderr, + ), + 1, + ) + self.assertEqual(stdout.getvalue(), "") + self.assertEqual(stderr.getvalue(), "ERROR git-dag-lab: reviewed failure\n") + + +class PackFileBoundaryTests(unittest.TestCase): + def test_reads_one_regular_single_link_file(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "pack" + path.write_bytes(b"bounded") + self.assertEqual( + pack_module._read_regular_file(path, label="fixture"), + b"bounded", + ) + + @unittest.skipUnless(hasattr(os, "symlink"), "symlink support is required") + def test_rejects_symlink_and_hardlink_inputs(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + target = root / "target" + target.write_bytes(b"bounded") + symlink = root / "symlink" + symlink.symlink_to(target.name) + hardlink = root / "hardlink" + os.link(target, hardlink) + for path in (symlink, target, hardlink): + with self.subTest(path=path.name), self.assertRaises( + VerificationError + ): + pack_module._read_regular_file(path, label="fixture") + + def test_rejects_empty_and_oversized_files(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + empty = root / "empty" + empty.write_bytes(b"") + oversized = root / "oversized" + with oversized.open("wb") as stream: + stream.truncate(MAX_PACK_BYTES + 1) + for path in (empty, oversized): + with self.assertRaises(VerificationError): + pack_module._read_regular_file(path, label="fixture") + + +if __name__ == "__main__": + unittest.main() From b12bc07b36327b4413034bcf7efacdb8364a5182 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:09:03 +0000 Subject: [PATCH 02/12] Add source-bound pack evidence pipeline --- .github/workflows/ci.yml | 13 +- .github/workflows/stage-pack-evidence.yml | 86 +++ README.md | 50 +- SECURITY.md | 8 +- git_dag_lab/pack.py | 1 + tests/test_pack_evidence.py | 228 +++++++ tools/capture_pack_report.sh | 325 +++++++++ tools/generate_pack_evidence.py | 775 ++++++++++++++++++++++ 8 files changed, 1476 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/stage-pack-evidence.yml create mode 100644 tests/test_pack_evidence.py create mode 100755 tools/capture_pack_report.sh create mode 100755 tools/generate_pack_evidence.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25eea94..29dd7ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,10 @@ jobs: cmp docs/demo/git-dag-v1/verify.txt "$RUNNER_TEMP/verify.txt" env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab inspect --compact > "$RUNNER_TEMP/inspect.json" cmp evidence/git-dag-v1.json "$RUNNER_TEMP/inspect.json" + env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab pack-verify > "$RUNNER_TEMP/pack-verify.txt" + cmp docs/demo/git-pack-index-v1/verify.txt "$RUNNER_TEMP/pack-verify.txt" + env -u PYTHONHOME -u PYTHONPATH python -B -m git_dag_lab pack-inspect --compact > "$RUNNER_TEMP/pack-inspect.json" + cmp evidence/git-pack-index-v1.json "$RUNNER_TEMP/pack-inspect.json" - name: Check tests, evidence freshness, and shell boundary shell: bash @@ -71,9 +75,11 @@ jobs: set -euo pipefail env -u PYTHONHOME -u PYTHONPATH python -W error -m unittest discover -s tests -v env -u PYTHONHOME -u PYTHONPATH python -B tools/generate_evidence.py --check + env -u PYTHONHOME -u PYTHONPATH python -B tools/generate_pack_evidence.py --check bash -n tools/capture_report.sh + bash -n tools/capture_pack_report.sh test "$(shellcheck --version | awk '$1 == "version:" {print $2}')" = "0.9.0" - shellcheck tools/capture_report.sh + shellcheck tools/capture_report.sh tools/capture_pack_report.sh test "$(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none)" = "" distribution: @@ -119,6 +125,11 @@ jobs: env -u PYTHONHOME -u PYTHONPATH "$runtime/bin/python" -B -m git_dag_lab verify ) > "$RUNNER_TEMP/wheel-verify.txt" cmp docs/demo/git-dag-v1/verify.txt "$RUNNER_TEMP/wheel-verify.txt" + ( + cd "$RUNNER_TEMP" + env -u PYTHONHOME -u PYTHONPATH "$runtime/bin/python" -B -m git_dag_lab pack-verify + ) > "$RUNNER_TEMP/wheel-pack-verify.txt" + cmp docs/demo/git-pack-index-v1/verify.txt "$RUNNER_TEMP/wheel-pack-verify.txt" test "$(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none)" = "" - name: Upload verified distributions diff --git a/.github/workflows/stage-pack-evidence.yml b/.github/workflows/stage-pack-evidence.yml new file mode 100644 index 0000000..71e4088 --- /dev/null +++ b/.github/workflows/stage-pack-evidence.yml @@ -0,0 +1,86 @@ +name: Stage pack evidence (temporary) + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: stage-pack-evidence-${{ github.ref }} + cancel-in-progress: false + +env: + PYTHONDONTWRITEBYTECODE: "1" + PYTHONNOUSERSITE: "1" + PYTHONHASHSEED: "0" + TZ: UTC + +jobs: + stage: + name: Generate read-only pack evidence artifact + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Check out the exact feature revision + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + ref: ${{ github.sha }} + + - name: Set up exact Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.12.12" + + - name: Assert the bounded staging context + shell: bash + run: | + set -euo pipefail + test "$GITHUB_REPOSITORY" = "omar07ibrahim/hello-git" + test "$GITHUB_EVENT_NAME" = "workflow_dispatch" + test "$GITHUB_REF" = "refs/heads/agent/pack-index-inspector" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + test "$(python -c 'import platform; print(platform.python_version())')" = "3.12.12" + test -z "${OPENAI_API_KEY:-}" + test "$(git status --porcelain=v1 --untracked-files=normal --ignore-submodules=none)" = "" + + - name: Acquire the digest-pinned browser image + shell: bash + run: | + set -euo pipefail + image='mcr.microsoft.com/playwright@sha256:2f29369043d81d6d69a815ceb80760f55e85f5020371ad06a4d996f18503ad1c' + docker pull "$image" + test "$(docker image inspect --format '{{index .RepoDigests 0}}' "$image")" = "$image" + + - name: Generate and independently replay both evidence packages + shell: bash + run: | + set -euo pipefail + tools/capture_pack_report.sh + python -B tools/generate_evidence.py --write + python -B tools/generate_evidence.py --check + python -B tools/generate_pack_evidence.py --check + python -W error -m unittest discover -s tests -v + bash -n tools/capture_pack_report.sh + test "$(shellcheck --version | awk '$1 == "version:" {print $2}')" = "0.9.0" + shellcheck tools/capture_pack_report.sh + + - name: Upload generated evidence without repository write access + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: git-pack-index-evidence-${{ github.sha }} + path: | + docs/assets/git-pack-cli.svg + docs/assets/git-pack-fanout.svg + docs/assets/git-pack-integrity.svg + docs/assets/git-pack-layout.svg + docs/assets/git-pack-report.png + docs/demo/git-dag-v1/manifest.json + docs/demo/git-pack-index-v1/ + evidence/git-pack-index-v1.json + if-no-files-found: error + include-hidden-files: false + compression-level: 0 + retention-days: 1 diff --git a/README.md b/README.md index 0273555..e7ecd8a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Git DAG Evidence Lab -> A dependency-free Python systems lab that constructs a real Git object database from plumbing commands and independently verifies every byte-addressed object. +> A dependency-free Python systems lab that constructs real Git object storage, then independently verifies loose-object DAGs plus closed-subset pack v2/index v2 bytes. The experiment demonstrates a subtle but important property: a merge commit and a rebase-shaped replay can resolve to **exactly the same tree** while preserving **different histories**. @@ -15,6 +15,8 @@ Requirements: Python 3.10+ and Git 2.29+. There are no runtime Python dependenci ```bash python3 -m git_dag_lab verify python3 -m git_dag_lab inspect +python3 -m git_dag_lab pack-verify +python3 -m git_dag_lab pack-inspect python3 -m unittest discover -s tests -v ``` @@ -34,6 +36,27 @@ The fixture contains three blobs, six trees, and five commits. `merge` stores th The replay is called **rebase-shaped** because it is constructed directly with `git commit-tree`. The lab does not claim to execute porcelain `git rebase`. + +## A second systems slice: verify pack and index bytes + +The pack path stores three fixed synthetic blobs in a fresh private SHA-1 repository, invokes real `git pack-objects`, then removes the repository after independently decoding both generated files. The CLI receipt below is actual stdout from that production path: + + + +The offline report is rendered from the same canonical receipt and captured by digest-pinned Chromium in a read-only, network-disabled container: + + + +The verifier does not trust the pack filename or Git's index. It parses the variable-length pack entry headers, bounds each zlib stream, reconstructs logical blob IDs, verifies the pack trailer, then parses the 256-entry cumulative fanout table, sorted OIDs, CRC32 rows, 32/64-bit offsets, pack binding, and index checksum. + + + + + + + +This is deliberately a closed subset: pack v2 and index v2, at most 64 objects, 1 MiB files, 256 KiB expanded objects, and non-delta entries only. OFS/REF deltas, other object formats, arbitrary repositories, reachability, and pack optimization are not claimed. SHA-1 and CRC32 model Git storage integrity here; neither is presented as authentication, a signature, or collision-resistant security. + ## The hard part: verify Git without trusting Git Writing an object with Git and asking Git to identify it would only prove that Git agrees with itself. This lab reads the raw stored bytes and independently computes: @@ -51,7 +74,7 @@ SHA-1 is used because this scenario models a SHA-1 Git object database. Here it ## Isolation and execution boundaries - Git is resolved once to an absolute executable and invoked with argument arrays, never a shell. -- 10 local subcommands are allow-listed, while one fixed isolated `git init` creates the bare database; transport commands and remote-looking arguments are rejected. +- 11 local subcommands are allow-listed, including the bounded `pack-objects` path, while one fixed isolated `git init` creates each bare database; transport commands and remote-looking arguments are rejected. - `HOME`, `XDG_CONFIG_HOME`, and `TMPDIR` are private; inherited Git config, hooks, replacement objects, identity, and object-directory redirects are ignored. - fixed synthetic identity `dag-lab@example.invalid`, fixed UTC timestamps, and fixed LF payloads make object IDs reproducible. - symlinked workspace components are rejected; stdout/stderr are spooled privately and checked before bounded reads. @@ -61,24 +84,26 @@ See [SECURITY.md](SECURITY.md) for the threat model and trusted-input boundary. ## Evidence pipeline -Every README visual begins with the same canonical CLI document. The generator runs fresh experiments twice, requires byte-identical outputs, derives the SVG and offline HTML, and binds each artifact into a hash manifest. Digest-pinned Chromium then captures the report in a read-only container with `--network none`. A separate attestation binds the exact report, rendered DOM, PNG, browser binary/version, container digest, isolation policy, viewport, and capture-script hash; without that attestation, the generator refuses to call the screenshot verified. +Every README visual begins with a canonical production CLI document. The DAG and pack generators each run fresh experiments twice, require byte-identical outputs, derive their SVGs and offline HTML, and bind every artifact into a hash manifest. Digest-pinned Chromium captures both reports in read-only containers with `--network none`. Separate attestations bind each exact report, rendered DOM, PNG, browser binary/version, container digest, isolation policy, viewport, and capture-script hash; without the matching attestation, a generator refuses to call its screenshot verified.  ### Reproduce the checked-in evidence ```bash -# Verify JSON, transcripts, SVGs, HTML, source hashes, and the existing PNG. +# Verify both JSON/transcript/visual/report/manifest packages and PNG attestations. python3 -B tools/generate_evidence.py --check +python3 -B tools/generate_pack_evidence.py --check -# Rebuild evidence and recapture the report with the pinned browser container. +# Rebuild and recapture either offline report with pinned Chromium. tools/capture_report.sh +tools/capture_pack_report.sh -# Run all engine, boundary, CLI, evidence, and provenance tests. +# Run all parser, boundary, CLI, evidence, and provenance tests. python3 -W error -m unittest discover -s tests -v ``` -Current verified baseline: **61 tests**, **9/9 graph invariants**, **57 isolated Git invocations**, report receipt `2da1ccd8…69c84`, and screenshot SHA-256 `e539db11…e17e`. +Current verified baseline: **89 tests**, **9/9 graph invariants**, **7/7 pack/index checks**, 57 isolated Git invocations in the DAG evidence run, two independently replayed evidence packages, and two attested offline browser captures. | Artifact | What it proves | |---|---| @@ -89,13 +114,22 @@ Current verified baseline: **61 tests**, **9/9 graph invariants**, **57 isolated | [`rendered-dom.html`](docs/demo/git-dag-v1/rendered-dom.html) | Actual DOM emitted by Chromium during the attested capture | | [`capture-attestation.json`](docs/demo/git-dag-v1/capture-attestation.json) | Report/DOM/PNG hashes plus verified browser, container, isolation, viewport, and script provenance | | [`manifest.json`](docs/demo/git-dag-v1/manifest.json) | SHA-256, byte size, role, source hashes, normalized argv, and attestation receipt | -| [`git-dag-report.png`](docs/assets/git-dag-report.png) | Actual Chromium rendering of the report at 1440×1800 | +| [`git-dag-report.png`](docs/assets/git-dag-report.png) | Actual Chromium rendering of the DAG report at 1440×1800 | +| [`evidence/git-pack-index-v1.json`](evidence/git-pack-index-v1.json) | Canonical real pack/index receipt, physical entry order, cross-bound rows, and non-claims | +| [`git-pack-cli.svg`](docs/assets/git-pack-cli.svg) | Exact production `pack-verify` stdout rendered as an accessible terminal panel | +| [`git-pack-layout.svg`](docs/assets/git-pack-layout.svg) | Actual pack offsets/sizes and index-table byte counts | +| [`git-pack-fanout.svg`](docs/assets/git-pack-fanout.svg) | Actual non-empty fanout buckets and sorted OID ranges | +| [`git-pack-integrity.svg`](docs/assets/git-pack-integrity.svg) | Receipt-derived pack/index checksum and row-binding workflow | +| [`git-pack-report.png`](docs/assets/git-pack-report.png) | Actual Chromium rendering of the pack/index report at 1440×1500 | +| [`git-pack-index-v1/manifest.json`](docs/demo/git-pack-index-v1/manifest.json) | Hash/size/source/command/capture inventory for every pack visual and output | ## Test coverage by risk The standard-library suite exercises more than happy-path graph construction: - independent blob, tree, and commit envelope hashes; +- pack v2 headers, bounded zlib streams, logical OIDs, trailer checksum, and explicit delta rejection; +- index v2 fanout, sorted OIDs, CRC32 rows, small/large offsets, pack binding, and checksum mutations; - exact object/ref inventories, parent ordering, reachability, and ancestry; - Git's special `directory/` tree ordering, truncated binary objects, and malformed headers; - hostile inherited Git environment and fake global identity/config; diff --git a/SECURITY.md b/SECURITY.md index e4cc6f4..37c677e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security model -Git DAG Evidence Lab creates a temporary bare repository below the selected workspace and removes it after each run. It does not inspect the repository that contains this source code. +Git DAG Evidence Lab creates a temporary bare repository below the selected workspace for each DAG or pack experiment and removes it after the run. It does not inspect the repository that contains this source code. The command boundary is deliberately narrow: @@ -13,6 +13,12 @@ The command boundary is deliberately narrow: - temporary roots with symlinked path components are rejected. - stdout and stderr are captured in private temporary files and rejected before loading into memory when either exceeds 1 MiB. +## Pack/index closed subset + +The pack experiment passes only three fixed synthetic blob IDs to `git pack-objects`; it does not accept a repository path, revision, ref, or caller-provided object list. Generated `.pack` and `.idx` files must be regular, single-link files no larger than 1 MiB and must remain the same inode and size across the bounded read. + +The independent parser accepts pack v2 and index v2 only. It rejects OFS/REF deltas, more than 64 objects, objects expanding beyond 256 KiB, invalid or unterminated zlib streams, duplicate logical objects, non-canonical fanout/large-offset tables, and any mismatch among logical object IDs, CRC32 rows, offsets, pack trailer, index pack binding, or index checksum. These checks establish the fixed fixture's storage integrity; they do not establish provenance, authenticity, repository reachability, or safety of arbitrary Git data. + ## SHA-1 scope The lab uses SHA-1 because the scenario explicitly models a SHA-1 Git object database. The independent envelope calculation demonstrates deterministic content addressing and detects accidental changes in these fixtures. It is not a signature, authentication mechanism, or claim of modern collision resistance. diff --git a/git_dag_lab/pack.py b/git_dag_lab/pack.py index 0da23da..606b50b 100644 --- a/git_dag_lab/pack.py +++ b/git_dag_lab/pack.py @@ -459,6 +459,7 @@ def _build_and_verify_pack(runner: _GitRunner, private: Path) -> PackReport: "pack-objects", "--window=0", "--depth=0", + "--compression=0", "--no-reuse-delta", "--no-reuse-object", os.fspath(prefix), diff --git a/tests/test_pack_evidence.py b/tests/test_pack_evidence.py new file mode 100644 index 0000000..513aa31 --- /dev/null +++ b/tests/test_pack_evidence.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import re +import unittest + +from tools import generate_pack_evidence as evidence + +ROOT = Path(__file__).resolve().parents[1] + + +class PackEvidenceTests(unittest.TestCase): + def test_expected_generated_inventory_is_exact(self) -> None: + generated = evidence.build_artifacts(allow_missing_screenshot=False) + self.assertEqual( + set(generated), + { + evidence.EVIDENCE_PATH, + evidence.VERIFY_PATH, + evidence.INSPECT_PATH, + evidence.REPORT_PATH, + evidence.MANIFEST_PATH, + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + }, + ) + + def test_checked_in_generated_files_are_current(self) -> None: + generated = evidence.build_artifacts(allow_missing_screenshot=False) + for path, expected in generated.items(): + with self.subTest(path=path.as_posix()): + self.assertEqual((ROOT / path).read_bytes(), expected) + + def test_compact_pretty_and_verify_outputs_share_one_receipt(self) -> None: + compact = json.loads((ROOT / evidence.EVIDENCE_PATH).read_text()) + pretty = json.loads((ROOT / evidence.INSPECT_PATH).read_text()) + verify = (ROOT / evidence.VERIFY_PATH).read_text() + self.assertEqual(compact, pretty) + self.assertIn( + f"receipt_sha256={compact['receipt']['sha256']}", + verify, + ) + canonical = json.dumps( + compact, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + b"\n" + self.assertEqual((ROOT / evidence.EVIDENCE_PATH).read_bytes(), canonical) + + def test_two_fresh_evidence_builds_are_byte_identical(self) -> None: + first = evidence.build_artifacts(allow_missing_screenshot=False) + second = evidence.build_artifacts(allow_missing_screenshot=False) + self.assertEqual(first, second) + + def test_manifest_binds_artifacts_capture_and_sources(self) -> None: + manifest = json.loads((ROOT / evidence.MANIFEST_PATH).read_text()) + rows = {row["path"]: row for row in manifest["artifacts"]} + expected = { + evidence.EVIDENCE_PATH, + evidence.VERIFY_PATH, + evidence.INSPECT_PATH, + evidence.REPORT_PATH, + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + evidence.SCREENSHOT_PATH, + evidence.RENDERED_DOM_PATH, + evidence.ATTESTATION_PATH, + } + self.assertEqual(set(rows), {path.as_posix() for path in expected}) + for path in expected: + content = (ROOT / path).read_bytes() + row = rows[path.as_posix()] + self.assertEqual(row["size"], len(content)) + self.assertEqual(row["sha256"], hashlib.sha256(content).hexdigest()) + self.assertEqual(manifest["capture"]["status"], "attested") + sources = {row["path"]: row for row in manifest["sources"]} + self.assertEqual( + set(sources), + { + "git_dag_lab/pack.py", + "git_dag_lab/cli.py", + "tools/generate_pack_evidence.py", + "tools/capture_pack_report.sh", + }, + ) + for relative, row in sources.items(): + content = (ROOT / relative).read_bytes() + self.assertEqual(row["size"], len(content)) + self.assertEqual(row["sha256"], hashlib.sha256(content).hexdigest()) + + def test_svg_visuals_are_accessible_and_receipt_bound(self) -> None: + document = json.loads((ROOT / evidence.EVIDENCE_PATH).read_text()) + receipt = document["receipt"]["sha256"] + for path in ( + evidence.LAYOUT_PATH, + evidence.FANOUT_PATH, + evidence.INTEGRITY_PATH, + evidence.CLI_PATH, + ): + text = (ROOT / path).read_text() + with self.subTest(path=path.as_posix()): + self.assertIn("