diff --git a/docs/changelog/3278.bugfix.rst b/docs/changelog/3278.bugfix.rst new file mode 100644 index 000000000..4d714c59f --- /dev/null +++ b/docs/changelog/3278.bugfix.rst @@ -0,0 +1 @@ +Correct SBOM CSV parsing and unresolved dependency relationships, and omit CI run identifiers from reproducible builds. diff --git a/hatch_build.py b/hatch_build.py index e48bde0a7..8d5231756 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -2,9 +2,9 @@ import ast import base64 +import csv import hashlib import json -import os import platform import re import shutil @@ -17,11 +17,13 @@ from email.parser import Parser from email.utils import getaddresses from importlib.metadata import distributions +from io import StringIO from itertools import starmap from pathlib import Path from typing import TYPE_CHECKING, Any, Final from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from hatchling.builders.utils import get_reproducible_timestamp from packaging.requirements import Requirement if TYPE_CHECKING: @@ -69,17 +71,6 @@ "sha384": "SHA-384", "sha512": "SHA-512", } -# only these names are ever read from the environment: they identify the CI run and carry no secrets -_GITHUB_PROVENANCE: Final[tuple[str, ...]] = ( - "GITHUB_REPOSITORY", - "GITHUB_SHA", - "GITHUB_REF", - "GITHUB_WORKFLOW", - "GITHUB_RUN_ID", - "GITHUB_RUN_ATTEMPT", - "RUNNER_OS", - "RUNNER_ARCH", -) class SbomBuildHook(BuildHookInterface): @@ -93,7 +84,8 @@ class SbomBuildHook(BuildHookInterface): The document also records the build environment (interpreter, OS, every distribution in the isolated build env with its files and the dependency graph between them), which PEP 770 calls out as what a third party needs to verify - build reproducibility, plus the source revision and the CI run that produced the wheel when known. + build reproducibility, plus the source revision when known. Release attestations identify the CI run without + introducing run-specific values into the wheel. """ @@ -140,7 +132,6 @@ def _cyclonedx_document(core: CoreMetadata, version: str) -> dict[str, Any]: "components": [*bundled, *declared], "dependencies": [ {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]]}, - *({"ref": component["bom-ref"], "dependsOn": []} for component in [*bundled, *declared]), *tool_dependencies, ], # transitive runtime dependencies are unknown until install time @@ -179,11 +170,6 @@ def _root_component(core: CoreMetadata, version: str) -> dict[str, Any]: if commit := _commit(): references.append({"type": "vcs", "url": f"{_REPOSITORY}/tree/{commit}", "comment": "exact source revision"}) properties.append({"name": "virtualenv:vcs-commit", "value": commit}) - if run_id := os.environ.get("GITHUB_RUN_ID"): - server = os.environ.get("GITHUB_SERVER_URL", "https://github.com") - references.append( - {"type": "build-system", "url": f"{server}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{run_id}"}, - ) return { "type": "application", "bom-ref": purl, @@ -211,8 +197,6 @@ def _external_reference(label: str, url: str) -> dict[str, str]: def _commit() -> str | None: - if commit := os.environ.get("GITHUB_SHA"): - return commit if not (_ROOT / ".git").exists() or (git := shutil.which("git")) is None: # building from an sdist return None return subprocess.run( @@ -265,7 +249,7 @@ def _bundled_component(wheel: Path) -> dict[str, Any]: } component["components"] = [ _file_component(component["bom-ref"], path, digest, size) - for path, digest, size in (line.split(",") for line in record.splitlines() if line) + for path, digest, size in (row for row in csv.reader(StringIO(record, newline="")) if row) if digest ] return component @@ -377,7 +361,8 @@ def _build_tools(package_version: str) -> tuple[list[dict[str, Any]], list[dict[ "properties": [ {"name": "python:implementation", "value": platform.python_implementation()}, {"name": "python:compiler", "value": platform.python_compiler()}, - {"name": "python:build", "value": " ".join(platform.python_build())}, + # GraalPy may omit the build date instead of returning an empty string. + {"name": "python:build", "value": " ".join(part for part in platform.python_build() if part)}, ], }, _operating_system(), @@ -443,18 +428,11 @@ def _workflow(root: dict[str, Any], tools: list[dict[str, Any]]) -> dict[str, An "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], } - if recorded := [ - {"name": name, "value": os.environ[name]} - for name in ("SOURCE_DATE_EPOCH", *_GITHUB_PROVENANCE) - if name in os.environ - ]: - workflow["inputs"] = [{"environmentVars": recorded}] + workflow["inputs"] = [ + {"environmentVars": [{"name": "SOURCE_DATE_EPOCH", "value": str(get_reproducible_timestamp())}]}, + ] return workflow def _timestamp() -> str: - # SOURCE_DATE_EPOCH makes the document reproducible; without it the real creation time is the truthful value, - # not hatchling's fixed 2020 fallback - if (epoch := os.environ.get("SOURCE_DATE_EPOCH")) is not None: - return datetime.fromtimestamp(int(epoch), tz=timezone.utc).isoformat() - return datetime.now(tz=timezone.utc).replace(microsecond=0).isoformat() + return datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).isoformat() diff --git a/pyproject.toml b/pyproject.toml index 2073fac03..d0a7c6b85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,6 +109,7 @@ test = [ "covdefaults>=2.3", "coverage>=7.2.7", "coverage-enable-subprocess>=1", + "hatchling>=1.27", "packaging>=23.1", "pytest>=7.4", "pytest-env>=0.8.2", diff --git a/tasks/validate_sbom.py b/tasks/validate_sbom.py index 786eaac53..71aa39531 100644 --- a/tasks/validate_sbom.py +++ b/tasks/validate_sbom.py @@ -87,8 +87,6 @@ def _validate_references(document: dict[str, Any]) -> list[str]: problems.append( f"root dependsOn {sorted(dependencies.get(root_ref, []))} != components {sorted(component_refs)}" ) - if missing := component_refs - set(dependencies): - problems.append(f"components without a dependencies entry: {sorted(missing)}") for workflow in document["formulation"][0]["workflows"]: referenced = {reference["ref"] for reference in workflow["resourceReferences"]} diff --git a/tests/unit/test_sbom.py b/tests/unit/test_sbom.py new file mode 100644 index 000000000..2660a6061 --- /dev/null +++ b/tests/unit/test_sbom.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import csv +import json +import shutil +import zipfile +from io import StringIO +from pathlib import Path +from typing import TYPE_CHECKING, Final + +import pytest +from hatchling.builders.wheel import WheelBuilder + +if TYPE_CHECKING: + from collections.abc import Callable + + from pytest_mock import MockerFixture + + +@pytest.fixture +def build_sbom(tmp_path: Path) -> Callable[[str], str]: + shutil.copyfile(Path(__file__).parents[2] / "hatch_build.py", tmp_path / "hatch_build.py") + (tmp_path / "LICENSE").write_text("Copyright (c) example\n", encoding="utf-8") + embed: Final[Path] = tmp_path / "src" / "virtualenv" / "seed" / "wheels" / "embed" + embed.mkdir(parents=True) + (embed / "__init__.py").write_text( + 'BUNDLE_SUPPORT = {"3.14": {"pip": "pip-1.0-py3-none-any.whl"}}\n', encoding="utf-8" + ) + + def build(filename: str) -> str: + record: Final[StringIO] = StringIO(newline="") + csv.writer(record).writerows([ + (filename, "sha256=ungWv48Bz-pBQUDeXa4iI7ADYaOWF3qctBD_YfIAFa0", "3"), + ("pip-1.0.dist-info/RECORD", "", ""), + ]) + with zipfile.ZipFile(embed / "pip-1.0-py3-none-any.whl", "w") as archive: + archive.writestr( + zipfile.ZipInfo("pip-1.0.dist-info/METADATA"), "Metadata-Version: 2.4\nName: pip\nVersion: 1.0\n" + ) + archive.writestr(zipfile.ZipInfo("pip-1.0.dist-info/RECORD"), record.getvalue()) + archive.writestr(zipfile.ZipInfo(filename), b"abc") + builder: Final[WheelBuilder] = WheelBuilder( + str(tmp_path), + config={ + "project": { + "name": "virtualenv", + "version": "1.0", + "description": "example", + "requires-python": ">=3.9", + "license": "MIT", + "maintainers": [{"name": "Example"}], + "dependencies": ["python-discovery>=1.6"], + }, + "tool": {"hatch": {"build": {"hooks": {"custom": {"path": "hatch_build.py"}}}}}, + }, + ) + build_data: Final = builder.get_default_build_data() + if "sbom_files" not in build_data: + pytest.skip("Hatchling before 1.28 does not support SBOMs (Python 3.9 builds)") + builder.get_build_hooks(str(tmp_path))["custom"].initialize("standard", build_data) + return Path(build_data["sbom_files"][0]).read_text(encoding="utf-8") + + return build + + +@pytest.mark.parametrize( + "filename", + [ + pytest.param("pip/example.py", id="plain"), + pytest.param("pip/a,b.py", id="comma"), + pytest.param('pip/a"b.py', id="quote"), + pytest.param("pip/a\nb.py", id="newline"), + ], +) +def test_sbom_record_paths(build_sbom: Callable[[str], str], filename: str) -> None: + document = json.loads(build_sbom(filename)) + assert document["components"][0]["components"] == [ + { + "type": "file", + "bom-ref": f"pkg:pypi/pip@1.0#{filename}", + "name": filename, + "hashes": [ + {"alg": "SHA-256", "content": "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"} + ], + "properties": [{"name": "size", "value": "3"}], + }, + ] + + +def test_sbom_unresolved_dependencies(build_sbom: Callable[[str], str]) -> None: + document = json.loads(build_sbom("pip/example.py")) + assert [entry for entry in document["dependencies"] if not entry["ref"].startswith("tool:")] == [ + {"ref": "pkg:pypi/virtualenv@1.0", "dependsOn": ["pkg:pypi/pip@1.0", "requires-dist:python-discovery>=1.6"]}, + ] + + +@pytest.mark.parametrize( + ("epoch", "expected"), + [ + pytest.param(None, "2020-02-02T00:00:00+00:00", id="hatchling-default"), + pytest.param("1700000000", "2023-11-14T22:13:20+00:00", id="source-date-epoch"), + ], +) +def test_sbom_timestamp( + build_sbom: Callable[[str], str], monkeypatch: pytest.MonkeyPatch, epoch: str | None, expected: str +) -> None: + monkeypatch.delenv("SOURCE_DATE_EPOCH", raising=False) + if epoch is not None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", epoch) + assert json.loads(build_sbom("pip/example.py"))["metadata"]["timestamp"] == expected + + +@pytest.mark.parametrize( + ("build", "expected"), + [ + pytest.param(("main", "Sep 21 2026"), "main Sep 21 2026", id="complete"), + pytest.param(("main", None), "main", id="graalpy-missing-date"), + pytest.param(("main", ""), "main", id="empty-date"), + ], +) +def test_sbom_python_build( + build_sbom: Callable[[str], str], mocker: MockerFixture, build: tuple[str, str | None], expected: str +) -> None: + mocker.patch("platform.python_build", autospec=True, return_value=build) + document: Final = json.loads(build_sbom("pip/example.py")) + assert [ + prop["value"] + for component in document["metadata"]["tools"]["components"] + for prop in component.get("properties", []) + if prop["name"] == "python:build" + ] == [expected] + + +def test_sbom_ci_rerun(build_sbom: Callable[[str], str], monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("SOURCE_DATE_EPOCH", "1700000000") + monkeypatch.setenv("GITHUB_REPOSITORY", "downstream/project") + monkeypatch.setenv("GITHUB_SHA", "a" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "100") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + first: Final[str] = build_sbom("pip/example.py") + monkeypatch.setenv("GITHUB_SHA", "b" * 40) + monkeypatch.setenv("GITHUB_RUN_ID", "200") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + assert build_sbom("pip/example.py") == first