diff --git a/docs/changelog/3311.bugfix.rst b/docs/changelog/3311.bugfix.rst new file mode 100644 index 000000000..8ddfc6d54 --- /dev/null +++ b/docs/changelog/3311.bugfix.rst @@ -0,0 +1,3 @@ +Leave the build machine out of the wheel and zipapp SBOMs, so the wheel rebuilds byte for byte on any operating system +and architecture given the same source, ``SOURCE_DATE_EPOCH``, Python patch version and build backend versions. Zipapp +entries now carry the ``SOURCE_DATE_EPOCH`` timestamp and fixed permissions instead of the build time. diff --git a/docs/explanation.rst b/docs/explanation.rst index 22766163c..0e9789577 100644 --- a/docs/explanation.rst +++ b/docs/explanation.rst @@ -588,9 +588,10 @@ involved. **Reproducible builds** Provenance tells you which workflow run built a file, but you still trust that run. The release pins every timestamp - to ``SOURCE_DATE_EPOCH``, the commit time of the tag, so you can rebuild the sdist from the tag yourself and compare - the bytes. The wheel reproduces except for its SBOM, which describes the machine that built it, and the ``RECORD`` - entry that hashes the SBOM. + to ``SOURCE_DATE_EPOCH``, the commit time of the tag, so you can rebuild the sdist and the wheel from the tag + yourself and compare the bytes. The wheel's SBOM records the Python version and build backend the release used, + which a rebuild must match. It leaves out the machine that ran the build, which the provenance attestation already + names. **The SBOMs** Dependency scanners find the packages a project declares, and virtualenv declares neither ``pip`` nor diff --git a/docs/how-to/verify-release.rst b/docs/how-to/verify-release.rst index 5c4ce806e..b85f16cc4 100644 --- a/docs/how-to/verify-release.rst +++ b/docs/how-to/verify-release.rst @@ -145,9 +145,9 @@ List the distributions the zipapp bundles and the Python versions that load each $ jq -r '.components[] | "\(.name) \(.version) \([.properties[] | select(.name == "virtualenv:loaded-for-python").value] | join(","))"' \ virtualenv.pyz.cdx.json -******************* - Rebuild the sdist -******************* +***************************** + Rebuild the sdist and wheel +***************************** The release builds with ``SOURCE_DATE_EPOCH`` set to the commit time of the release tag, so rebuilding the tag yields the same sdist, byte for byte: @@ -159,5 +159,27 @@ the same sdist, byte for byte: $ SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) uv build --sdist --out-dir rebuild . $ cmp rebuild/virtualenv-21.10.0.tar.gz ../virtualenv-21.10.0.tar.gz -``cmp`` prints nothing when the files match. A rebuilt wheel matches the published one in every file except the SBOM, -which records the machine that built it, and ``RECORD``, which holds the SBOM's hash. +``cmp`` prints nothing when the files match. + +The wheel of a release after 21.10.0 rebuilds byte for byte too, on any operating system and architecture, once the +Python patch version and the build backend versions match the ones the release used. Its SBOM lists both, so read them +from the published wheel and pass them to the build: + +.. code-block:: console + + $ unzip -p ../virtualenv--py3-none-any.whl '*.dist-info/sboms/virtualenv.cdx.json' > published.cdx.json + $ jq -r '.metadata.tools.components[] | select(.type == "platform") | .version' published.cdx.json + 3.14.7 + $ jq -r '.metadata.tools.components[] | select(.purl // "" | startswith("pkg:pypi/")) | "\(.name)==\(.version)"' \ + published.cdx.json > build-constraints.txt + $ SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) uv build --wheel --python 3.14.7 \ + --build-constraint build-constraints.txt --out-dir rebuild . + $ cmp rebuild/virtualenv--py3-none-any.whl ../virtualenv--py3-none-any.whl + +Build from a git checkout, since the SBOM records the source commit and an sdist does not carry it. Wheels up to 21.10.0 +recorded the machine that built them in the SBOM, so a rebuild of those differs in the SBOM and in ``RECORD``, which +holds the SBOM's hash. + +The zipapp rebuilds byte for byte with ``SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) tox r -e zipapp`` on the Python +version its SBOM lists, but only while every package the build pulls from PyPI still resolves to the version the release +used. The zipapp build does not pin them; the zipapp SBOM and the wheel SBOM inside the zipapp list them. diff --git a/docs/reference/release-artifacts.rst b/docs/reference/release-artifacts.rst index 27331fa2b..ad5dbf255 100644 --- a/docs/reference/release-artifacts.rst +++ b/docs/reference/release-artifacts.rst @@ -96,8 +96,9 @@ Wheel SBOM - Components: every wheel bundled under ``virtualenv/seed/wheels/embed``, with its SHA-256, license, the packages it vendors, and a ``virtualenv:seeded-for-python`` property per Python version that receives it; plus the runtime dependencies declared in the wheel metadata, without versions, since the installer resolves those. -- Build record: the interpreter, operating system and build backend packages that produced the wheel, the source commit, - and the ``SOURCE_DATE_EPOCH`` used for timestamps. +- Build record: the Python version and build backend packages that produced the wheel, the source commit, and the + ``SOURCE_DATE_EPOCH`` used for timestamps. Releases up to 21.10.0 also recorded the operating system, architecture and + interpreter build of the build machine. - First release carrying it: 21.8.1. SPDX rendering @@ -118,8 +119,9 @@ Zipapp SBOM SHA-256; and each bundled distribution, such as ``filelock`` or ``platformdirs``, with its version, license, a ``virtualenv:loaded-for-python`` property per Python version that imports it, and a SHA-256 per file. Every file in the archive other than the SBOM appears in it. -- Build record: the interpreter, operating system and packages of the environment that built the zipapp, and the - ``SOURCE_DATE_EPOCH`` used for timestamps. +- Build record: the Python version and packages of the environment that built the zipapp, and the ``SOURCE_DATE_EPOCH`` + used for timestamps. Packages installed from platform-specific wheels appear without their files, which differ per + operating system and architecture. *************************** Embedded wheel advisories @@ -135,5 +137,22 @@ Python 3.10 or newer to avoid them. *********************** The release sets ``SOURCE_DATE_EPOCH`` to the commit time of the tag (``git log -1 --pretty=%ct``). Rebuilding the tag -with the same value reproduces the sdist byte for byte. A rebuilt wheel differs from the published one only in the SBOM, -which records the build machine, and in ``RECORD``, which holds the SBOM's hash. +with the same value reproduces the sdist byte for byte. + +The wheel, whose SBOM `hatch_build.py `_ writes, reproduces +byte for byte on any operating system and architecture when these inputs match the release: + +- the source tree, as a git checkout of the tag, since the SBOM records the commit; +- ``SOURCE_DATE_EPOCH``; +- the Python patch version, which the SBOM records; +- the versions of the build backend and its dependencies, which the SBOM lists. + +Any build frontend works, since the SBOM leaves out the installer metadata a frontend writes into the build environment. + +Wheels up to 21.10.0 recorded the build machine in their SBOM, so a rebuild of those differs in the SBOM and in +``RECORD``, which holds the SBOM's hash. + +The zipapp, built by `tasks/make_zipapp.py `_, needs +the same inputs, and its entries carry ``SOURCE_DATE_EPOCH`` as their timestamp and fixed permissions. Its build also +downloads the distributions it bundles and the backend for the wheel inside it from PyPI without pinning them, so a +rebuild matches only while those resolve to the versions the release used. diff --git a/hatch_build.py b/hatch_build.py index 3e0d1d4fe..2018e5f4a 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -84,10 +84,11 @@ class SbomBuildHook(BuildHookInterface): described from its own ``METADATA`` and ``RECORD`` and hashed from its bytes, so a wheel bump needs no separate SBOM update. - The document also records the build environment (interpreter, OS, every distribution in the isolated build env with - its files and the dependency graph between them), which PEP 770 calls out as what a third party needs to verify - build reproducibility, plus the source revision when known. Release attestations identify the CI run without - introducing run-specific values into the wheel. + The document also records the build toolchain (the Python version, every distribution in the isolated build env with + the files of the pure-Python ones, and the dependency graph between them), which PEP 770 calls out as what a third + party needs to verify build reproducibility, plus the source revision when known. Nothing about the build machine + goes in, so a rebuild with the same toolchain on another OS or architecture produces the same wheel; release + attestations identify the CI run and the builder instead. """ @@ -403,16 +404,9 @@ def build_tools(package_version: str) -> tuple[list[dict[str, Any]], list[dict[s "bom-ref": f"tool:{interpreter}", "name": sys.implementation.name, "version": platform.python_version(), - "description": sys.version, "purl": interpreter, - "properties": [ - {"name": "python:implementation", "value": platform.python_implementation()}, - {"name": "python:compiler", "value": platform.python_compiler()}, - # GraalPy may omit the build date instead of returning an empty string. - {"name": "python:build", "value": " ".join(part for part in platform.python_build() if part)}, - ], + "properties": [{"name": "python:implementation", "value": platform.python_implementation()}], }, - _operating_system(), ] # bom-refs are prefixed because the same distribution can be both a build tool and a bundled component installed = {_purl(distribution.metadata["Name"]): distribution for distribution in distributions()} @@ -420,42 +414,28 @@ def build_tools(package_version: str) -> tuple[list[dict[str, Any]], list[dict[s for distribution in (installed[key] for key in sorted(installed)): component = component_from_metadata(distribution.metadata, "library") component["bom-ref"] = f"tool:{component['purl']}" - component["components"] = [ - _file_component( - component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) - ) - for file in distribution.files or [] - # console-script launchers live outside site-packages and embed the build env's interpreter path in - # their shebang, so their hash differs on every build and says nothing about the distribution - if file.hash is not None and not file.as_posix().startswith("../") - ] + # a platform wheel installs files built for the build machine's OS and architecture, so listing them would + # tie the document to that machine + if Parser().parsestr(distribution.read_text("WHEEL") or "")["Root-Is-Purelib"] == "true": + component["components"] = [ + _file_component( + component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) + ) + for file in distribution.files or [] + # console-script launchers live outside site-packages and embed the build env's interpreter path in + # their shebang, so their hash differs on every build and says nothing about the distribution; the + # installer metadata names the build frontend that set up the env rather than the distribution + if file.hash is not None + and not file.as_posix().startswith("../") + and not ( + file.parent.suffix == ".dist-info" and file.name in {"INSTALLER", "REQUESTED", "direct_url.json"} + ) + ] tools.append(component) tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) return tools, tool_dependencies -def _operating_system() -> dict[str, Any]: - component: dict[str, Any] = { - "type": "operating-system", - "bom-ref": f"tool:os:{platform.system()}@{platform.release()}", - "name": platform.system(), - "version": platform.release(), - "description": platform.platform(), - "properties": [ - {"name": "machine", "value": platform.machine()}, - {"name": "kernel-version", "value": platform.version()}, - ], - } - try: - os_release = platform.freedesktop_os_release() - except OSError: # not a freedesktop system, e.g. macOS or Windows - return component - component["properties"] += [ - {"name": f"os-release:{key}", "value": value} for key, value in sorted(os_release.items()) - ] - return component - - def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: refs = set() for requirement in map(Requirement, distribution.requires or []): diff --git a/tasks/cyclonedx_to_spdx.py b/tasks/cyclonedx_to_spdx.py index 75642ffea..8375b0d32 100644 --- a/tasks/cyclonedx_to_spdx.py +++ b/tasks/cyclonedx_to_spdx.py @@ -122,7 +122,6 @@ class Spdx(TypedDict): _PURPOSES: Final[dict[str, str]] = { "application": "APPLICATION", "library": "LIBRARY", - "operating-system": "OPERATING-SYSTEM", "platform": "OTHER", } diff --git a/tasks/make_zipapp.py b/tasks/make_zipapp.py index b16d41e7a..d5dd95146 100644 --- a/tasks/make_zipapp.py +++ b/tasks/make_zipapp.py @@ -14,6 +14,7 @@ import zipfile from collections import defaultdict from dataclasses import dataclass +from datetime import datetime, timezone from pathlib import Path, PurePosixPath from tempfile import TemporaryDirectory from typing import TYPE_CHECKING, Any, Final @@ -128,10 +129,10 @@ def create_zipapp(dest: str, packages: dict[str, dict[str, dict[str, WheelForVer with zipfile.ZipFile(bio, "w") as zip_app: write_packages_to_zipapp(base, dist, modules, packages, zip_app) modules_json = json.dumps(modules, indent=2) - zip_app.writestr("modules.json", modules_json) + zip_app.writestr(_entry("modules.json"), modules_json) distributions_json = json.dumps(dist, indent=2) - zip_app.writestr("distributions.json", distributions_json) - zip_app.writestr("__main__.py", (HERE / "__main__zipapp.py").read_bytes()) + zip_app.writestr(_entry("distributions.json"), distributions_json) + zip_app.writestr(_entry("__main__.py"), (HERE / "__main__zipapp.py").read_bytes()) bio.seek(0) zipapp.create_archive(bio, dest) print(f"zipapp created at {dest} with size {os.path.getsize(dest) / 1024 / 1024:.2f}MB") # ruff:ignore[print] @@ -173,9 +174,21 @@ def write_packages_to_zipapp( # ruff:ignore[complex-structure, too-many-branche continue print(dest_str) # ruff:ignore[print] content = wheel_zip.read(filename) - zip_app.writestr(dest_str, content) + zip_app.writestr(_entry(dest_str), content) del content +def _entry(name: str) -> zipfile.ZipInfo: + # the build time and OS would otherwise end up in each entry header; 1580601600 is hatchling's fallback, so the + # entries share the timestamp of the SBOM appended after them + epoch: Final[int] = int(os.environ.get("SOURCE_DATE_EPOCH", "1580601600")) + entry: Final[zipfile.ZipInfo] = zipfile.ZipInfo( + name, datetime.fromtimestamp(epoch, tz=timezone.utc).timetuple()[:6] + ) + entry.create_system = 3 + entry.external_attr = 0o644 << 16 + return entry + + if __name__ == "__main__": main() diff --git a/tasks/zipapp_sbom.py b/tasks/zipapp_sbom.py index 730df8da9..5745d45bd 100644 --- a/tasks/zipapp_sbom.py +++ b/tasks/zipapp_sbom.py @@ -69,6 +69,9 @@ def main() -> None: entry: Final[zipfile.ZipInfo] = zipfile.ZipInfo( _SBOM_NAME, datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).timetuple()[:6] ) + # match the entries tasks/make_zipapp.py writes, whichever OS appends the SBOM + entry.create_system = 3 + entry.external_attr = 0o644 << 16 with zipfile.ZipFile(pyz, "a") as archive: archive.writestr(entry, content) diff --git a/tests/unit/test_sbom.py b/tests/unit/test_sbom.py index efe18b699..3ea75e561 100644 --- a/tests/unit/test_sbom.py +++ b/tests/unit/test_sbom.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import csv import hashlib import json @@ -145,28 +146,55 @@ def test_sbom_timestamp( assert json.loads(build_sbom("pip/example.py", {}))["metadata"]["timestamp"] == expected -@pytest.mark.parametrize( - ("build", "expected"), - [ - pytest.param(("main", "Sep 21 2026"), "main Sep 21 2026", id="complete"), - pytest.param(("main", None), "main", id="graalpy-missing-date"), - pytest.param(("main", ""), "main", id="empty-date"), - ], -) -def test_sbom_python_build( +@pytest.mark.skipif(sys.version_info < (3, 10), reason="platform.freedesktop_os_release is new in Python 3.10") +def test_sbom_build_host_independent( build_sbom: Callable[[str, dict[str, str]], str], + install_tool: Callable[[Path, str, bool, dict[str, bytes]], None], mocker: MockerFixture, - build: tuple[str, str | None], - expected: str, + tmp_path: Path, ) -> None: - mocker.patch("platform.python_build", autospec=True, return_value=build) - document: Final[dict[str, Any]] = json.loads(build_sbom("pip/example.py", {})) - assert [ - prop["value"] - for component in document["metadata"]["tools"]["components"] - for prop in component.get("properties", []) - if prop["name"] == "python:build" - ] == [expected] + sboms: Final[list[str]] = [] + for host, extension, installer, sys_version in ( + ( + { + "system": "Linux", + "release": "6.17.0-1022-azure", + "version": "#22~24.04.1-Ubuntu SMP", + "machine": "x86_64", + "platform": "Linux-6.17.0-1022-azure-x86_64-with-glibc2.39", + "python_compiler": "GCC 13.3.0", + "python_build": ("main", "Aug 5 2026 10:00:00"), + "freedesktop_os_release": {"ID": "ubuntu", "VERSION_ID": "24.04"}, + }, + "cpython-314-x86_64-linux-gnu.so", + b"uv\n", + "3.14.7 (main, Aug 5 2026, 10:00:00) [GCC 13.3.0]", + ), + ( + { + "system": "Linux", + "release": "7.0.12-linuxkit", + "version": "#1 SMP PREEMPT", + "machine": "aarch64", + "platform": "Linux-7.0.12-linuxkit-aarch64-with-glibc2.36", + "python_compiler": "Clang 20.1.4", + "python_build": ("main", "Sep 1 2026 08:00:00"), + "freedesktop_os_release": {"ID": "debian", "VERSION_ID": "12"}, + }, + "cpython-314-aarch64-linux-gnu.so", + b"pip\n", + "3.14.7 (main, Sep 1 2026, 08:00:00) [Clang 20.1.4]", + ), + ): + for name, value in host.items(): + mocker.patch(f"platform.{name}", autospec=True, return_value=value) + mocker.patch.object(sys, "version", sys_version) + install_tool(site := tmp_path / extension, "native", False, {f"native/_speedups.{extension}": b"\0"}) + install_tool(site, "pure", True, {"pure-1.0.dist-info/INSTALLER": installer}) + mocker.patch.object(sys, "path", [str(site), *sys.path]) + sboms.append(build_sbom("pip/example.py", {})) + mocker.stopall() + assert sboms[0] == sboms[1] def test_sbom_ci_rerun(build_sbom: Callable[[str, dict[str, str]], str], monkeypatch: pytest.MonkeyPatch) -> None: @@ -361,7 +389,7 @@ def test_sbom_zipapp_embeds_document(zipapp: Path, zipapp_sbom: str) -> None: @pytest.fixture def zipapp(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: if sys.version_info < (3, 10): - pytest.skip("hatch_build.py reads the OS release with platform.freedesktop_os_release, new in Python 3.10") + pytest.skip("zipfile.Path shares and then closes the handle of the archive it wraps before Python 3.10") wheel: Final[BytesIO] = BytesIO() with zipfile.ZipFile(wheel, "w") as archive: archive.writestr("pip-1.0.dist-info/METADATA", "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n") @@ -401,6 +429,25 @@ def render(cyclonedx: str) -> str: return render +@pytest.fixture +def install_tool() -> Callable[[Path, str, bool, dict[str, bytes]], None]: + def install(site: Path, name: str, purelib: bool, files: dict[str, bytes]) -> None: + record: Final[StringIO] = StringIO(newline="") + for path, content in files.items(): + (target := site / path).parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(content) + digest = base64.urlsafe_b64encode(hashlib.sha256(content).digest()).rstrip(b"=").decode() + csv.writer(record).writerow((path, f"sha256={digest}", len(content))) + (dist_info := site / f"{name}-1.0.dist-info").mkdir(exist_ok=True) + (dist_info / "METADATA").write_text(f"Metadata-Version: 2.4\nName: {name}\nVersion: 1.0\n", encoding="utf-8") + (dist_info / "WHEEL").write_text( + f"Wheel-Version: 1.0\nRoot-Is-Purelib: {str(purelib).lower()}\n", encoding="utf-8" + ) + (dist_info / "RECORD").write_text(record.getvalue(), encoding="utf-8", newline="") + + return install + + @pytest.fixture def build_sbom(tmp_path: Path) -> Callable[[str, dict[str, str]], str]: shutil.copyfile(Path(__file__).parents[2] / "hatch_build.py", tmp_path / "hatch_build.py") diff --git a/tox.toml b/tox.toml index f21fad78b..15502374a 100644 --- a/tox.toml +++ b/tox.toml @@ -215,6 +215,7 @@ base_python = [ "3.14" ] skip_install = true deps = [ "hatchling>=1.28", "packaging>=25" ] dependency_groups = [ "sbom" ] +pass_env = [ "SOURCE_DATE_EPOCH" ] set_env.PYTHONPATH = "{tox_root}" commands = [ [ "python", "tasks/make_zipapp.py" ],