From 895408353f9b2c653147a356034f6d8ac97a0ad3 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 19 Sep 2026 09:40:48 -0700 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8=20feat(build):=20make=20the=20emb?= =?UTF-8?q?edded=20SBOM=20exhaustive?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SBOM shipped so far described the bundled wheels and nothing else, and its metadata.tools entry carried a `vendor` key that is not valid on a CycloneDX component, so the document did not conform to the 1.6 schema. The hand-rolled validator could not see that; only a real schema validation can. Measured against CISA's 2024 baseline attributes the document also lacked an SBOM author, lifecycle, supplier, relationship completeness, and the primary component's license and copyright. PEP 770 names the build tools and environment as what a third party needs to verify reproducibility, and none of that was recorded either. The document now describes the root component from the project metadata and LICENSE, each bundled wheel from its own METADATA and bytes (no more hardcoded license), the declared runtime dependencies with their specifiers and markers, and every distribution in the isolated build environment along with the dependency graph between them, the interpreter and the OS. When built in GitHub Actions the run URL and commit are recorded on the formulation workflow. Only an explicit allowlist of environment variable names is ever read. CI now validates the document with cyclonedx-python-lib's strict 1.6 schema validator and checks that every bom-ref resolves, so a malformed document fails on the pull request rather than at release attestation. --- hatch_build.py | 358 +++++++++++++++++++++++++++++++++-------- pyproject.toml | 1 + tasks/validate_sbom.py | 79 ++++----- 3 files changed, 328 insertions(+), 110 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index 873be010b..0d9beb7fb 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -1,19 +1,68 @@ from __future__ import annotations import ast +import hashlib import json +import os +import platform +import re +import sys import tempfile import uuid +import zipfile from datetime import datetime, timezone +from email.parser import Parser +from email.utils import getaddresses +from importlib.metadata import distributions +from itertools import starmap from pathlib import Path -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final from hatchling.builders.hooks.plugin.interface import BuildHookInterface from hatchling.builders.utils import get_reproducible_timestamp +from packaging.requirements import Requirement + +if TYPE_CHECKING: + from importlib.metadata import Distribution, PackageMetadata + + from hatchling.metadata.core import CoreMetadata _ROOT: Final[Path] = Path(__file__).resolve().parent -_EMBED_INIT: Final[Path] = _ROOT / "src" / "virtualenv" / "seed" / "wheels" / "embed" / "__init__.py" -_SBOM_NAMESPACE: Final[uuid.UUID] = uuid.uuid5(uuid.NAMESPACE_URL, "https://github.com/pypa/virtualenv/sboms") +_EMBED: Final[Path] = _ROOT / "src" / "virtualenv" / "seed" / "wheels" / "embed" +_REPOSITORY: Final[str] = "https://github.com/pypa/virtualenv" +_SBOM_NAMESPACE: Final[uuid.UUID] = uuid.uuid5(uuid.NAMESPACE_URL, f"{_REPOSITORY}/sboms") +_PYPA: Final[dict[str, Any]] = {"name": "Python Packaging Authority", "url": ["https://www.pypa.io"]} +# the label normalization and mapping cyclonedx-py applies to Project-URL entries, plus "source", so the same +# metadata yields the same reference types whichever generator produced the document +_URL_LABEL_TO_REFERENCE_TYPE: Final[dict[str, str]] = { + "bugtracker": "issue-tracker", + "issuetracker": "issue-tracker", + "issues": "issue-tracker", + "bugreports": "issue-tracker", + "tracker": "issue-tracker", + "home": "website", + "homepage": "website", + "download": "distribution", + "documentation": "documentation", + "docs": "documentation", + "changelog": "release-notes", + "changes": "release-notes", + "source": "vcs", + "repository": "vcs", + "github": "vcs", + "chat": "chat", +} +# only these names are ever read from the environment: they identify the CI run and carry no secrets +_GITHUB_PROVENANCE: Final[tuple[str, ...]] = ( + "GITHUB_REPOSITORY", + "GITHUB_SHA", + "GITHUB_REF", + "GITHUB_WORKFLOW", + "GITHUB_RUN_ID", + "GITHUB_RUN_ATTEMPT", + "RUNNER_OS", + "RUNNER_ARCH", +) class SbomBuildHook(BuildHookInterface): @@ -21,9 +70,12 @@ class SbomBuildHook(BuildHookInterface): General-purpose SBOM scanners (syft, cyclonedx-py, GitHub's dependency graph) read declared dependency metadata or an installed environment, and virtualenv's bundled pip and setuptools wheels are neither: they are data files - embedded under ``src/virtualenv/seed/wheels/embed/``, invisible to every one of those tools. Declaring them here, - read straight out of the ``BUNDLE_SUPPORT``/``BUNDLE_SHA256`` tables that ``tasks/upgrade_wheels.py`` already - maintains, means the SBOM tracks a wheel bump automatically instead of needing its own update step. + embedded under ``src/virtualenv/seed/wheels/embed/``, invisible to every one of those tools. Each bundled wheel is + described from its own ``METADATA`` and hashed from its bytes, so a wheel bump needs no separate SBOM update. + + The document also records the build environment (interpreter, OS, every distribution in the isolated build env and + the dependency graph between them), which PEP 770 calls out as what a third party needs to verify build + reproducibility, plus the CI run that produced the wheel when built in GitHub Actions. """ @@ -35,7 +87,7 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: # ruff: # sbom_files only exists on hatchling>=1.28, which build-system.requires excludes for the # Python versions old enough to still need hatchling<1.28 for building at all return - document = _cyclonedx_document(self.metadata.version, self.metadata.core.name) + document = _cyclonedx_document(self.metadata.core, self.metadata.version) # written outside self.directory, which hatchling also uses as the final wheel output location, so a # stray copy here would sit next to the built artifacts and trip `twine check dist/*` out = Path(tempfile.mkdtemp(prefix="virtualenv-sbom-")) / "virtualenv.cdx.json" @@ -43,78 +95,252 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: # ruff: build_data["sbom_files"].append(str(out)) -def _cyclonedx_document(version: str, name: str) -> dict[str, Any]: - wheel_sha256 = _bundled_wheels() - components = [] - dependencies = [{"ref": f"pkg:pypi/{name}@{version}", "dependsOn": []}] - for filename, sha256 in sorted(wheel_sha256.items()): - distribution, wheel_version = filename.split("-")[:2] - purl = f"pkg:pypi/{distribution}@{wheel_version}" - components.append({ - "type": "library", - "name": distribution, - "version": wheel_version, - "purl": purl, - "bom-ref": purl, - "licenses": [{"license": {"id": "MIT"}}], - "hashes": [{"alg": "SHA-256", "content": sha256}], - "externalReferences": [ - {"type": "distribution", "url": f"https://pypi.org/project/{distribution}/{wheel_version}/"}, - ], +def _cyclonedx_document(core: CoreMetadata, version: str) -> dict[str, Any]: + root = _root_component(core, version) + bundled = [_bundled_component(wheel) for wheel in sorted(_EMBED.glob("*.whl"))] + declared = [_declared_dependency(requirement) for requirement in core.dependencies] + tools, tool_dependencies = _build_tools(version) + body = { + "metadata": { + "timestamp": datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).isoformat(), + "lifecycles": [{"phase": "build"}], + "tools": {"components": tools}, + "manufacturer": _PYPA, + "authors": _contacts(core.maintainers_data["name"], core.maintainers_data["email"]), + "supplier": _PYPA, + "component": root, "properties": [ - {"name": "virtualenv:bundled-wheel", "value": f"src/virtualenv/seed/wheels/embed/{filename}"}, + { + "name": "virtualenv:sbom:scope", + "value": "Components are the wheels bundled as data files under src/virtualenv/seed/wheels/embed" + " and the direct runtime dependencies declared in the wheel's core metadata. The latter carry" + " no version because they are resolved at install time, not at build time.", + }, ], - }) - dependencies[0]["dependsOn"].append(purl) - dependencies.append({"ref": purl, "dependsOn": []}) + }, + "components": [*bundled, *declared], + "dependencies": [ + {"ref": root["bom-ref"], "dependsOn": [component["bom-ref"] for component in [*bundled, *declared]]}, + *({"ref": component["bom-ref"], "dependsOn": []} for component in [*bundled, *declared]), + *tool_dependencies, + ], + # transitive runtime dependencies are unknown until install time + "compositions": [{"aggregate": "incomplete", "dependencies": [root["bom-ref"]]}], + "formulation": [{"bom-ref": "formula:wheel-build", "workflows": [_workflow(root, tools)]}], + } return { + "$schema": "http://cyclonedx.org/schema/bom-1.6.schema.json", "bomFormat": "CycloneDX", "specVersion": "1.6", - "serialNumber": _serial_number(name, version, wheel_sha256), + # derived from the content so identical documents share a serial and any difference gets a new one + "serialNumber": f"urn:uuid:{uuid.uuid5(_SBOM_NAMESPACE, json.dumps(body, sort_keys=True))}", "version": 1, - "metadata": { - "timestamp": _timestamp(), - "tools": { - "components": [ - {"type": "application", "name": "hatch_build.py", "vendor": "pypa"}, - ], - }, - "component": { - "type": "application", - "name": name, - "version": version, - "purl": f"pkg:pypi/{name}@{version}", - "bom-ref": f"pkg:pypi/{name}@{version}", - }, - }, - "components": components, - "dependencies": dependencies, + **body, + } + + +def _root_component(core: CoreMetadata, version: str) -> dict[str, Any]: + purl = _purl(core.name, version) + license_lines = (_ROOT / "LICENSE").read_text(encoding="utf-8").splitlines() + references = list(starmap(_external_reference, core.urls.items())) + references += [ + {"type": "distribution", "url": f"https://pypi.org/project/{core.name}/{version}/"}, + {"type": "release-notes", "url": "https://virtualenv.pypa.io/en/latest/changelog.html"}, + {"type": "security-contact", "url": f"{_REPOSITORY}/security/policy"}, + {"type": "advisories", "url": f"{_REPOSITORY}/security/advisories"}, + {"type": "license", "url": f"{_REPOSITORY}/blob/main/LICENSE"}, + ] + if run_id := os.environ.get("GITHUB_RUN_ID"): + server = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + references.append( + {"type": "build-system", "url": f"{server}/{os.environ['GITHUB_REPOSITORY']}/actions/runs/{run_id}"}, + ) + return { + "type": "application", + "bom-ref": purl, + "supplier": _PYPA, + "authors": _contacts(core.maintainers_data["name"], core.maintainers_data["email"]), + "name": core.name, + "version": version, + "description": core.description, + "licenses": [{"expression": core.license_expression, "acknowledgment": "declared"}], + "copyright": next(line for line in license_lines if line.startswith("Copyright")), + "purl": purl, + "externalReferences": references, + "properties": [{"name": "virtualenv:requires-python", "value": core.requires_python}], + } + + +def _purl(name: str, version: str | None = None) -> str: + normalized = re.sub(r"[-_.]+", "-", name).lower() + return f"pkg:pypi/{normalized}@{version}" if version else f"pkg:pypi/{normalized}" + + +def _contacts(names: list[str], addresses: list[str]) -> list[dict[str, str]]: + contacts = [{"name": name} for name in names] + for name, email in getaddresses(addresses): + contacts.append({key: value for key, value in (("name", name), ("email", email)) if value}) + return contacts + + +def _external_reference(label: str, url: str) -> dict[str, str]: + reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} + + +def _bundled_component(wheel: Path) -> dict[str, Any]: + with zipfile.ZipFile(wheel) as archive: + metadata_name = next(name for name in archive.namelist() if name.endswith(".dist-info/METADATA")) + metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) + component = _component_from_metadata(metadata, "library") + if any(reference["url"].startswith("https://github.com/pypa/") for reference in component["externalReferences"]): + component["supplier"] = _PYPA + component["hashes"] = [{"alg": "SHA-256", "content": hashlib.sha256(wheel.read_bytes()).hexdigest()}] + component["externalReferences"].append( + {"type": "distribution", "url": f"https://pypi.org/project/{metadata['Name']}/{metadata['Version']}/"}, + ) + component["properties"] = [ + {"name": "virtualenv:bundled-wheel", "value": wheel.relative_to(_ROOT).as_posix()}, + *( + {"name": "virtualenv:seeded-for-python", "value": python_version} + for python_version, wheels in _bundle_support().items() + if wheel.name in wheels.values() + ), + ] + if requires_python := metadata.get("Requires-Python"): + component["properties"].append({"name": "virtualenv:requires-python", "value": requires_python}) + return component + + +def _component_from_metadata(metadata: PackageMetadata, component_type: str) -> dict[str, Any]: + name, version = metadata["Name"], metadata["Version"] + purl = _purl(name, version) + component: dict[str, Any] = { + "type": component_type, + "bom-ref": purl, + "name": name, + "version": version, + "purl": purl, } + if summary := metadata.get("Summary"): + component["description"] = summary + if authors := _contacts( + metadata.get_all("Author", []) + metadata.get_all("Maintainer", []), + metadata.get_all("Author-email", []) + metadata.get_all("Maintainer-email", []), + ): + component["authors"] = authors + if licenses := _licenses(metadata): + component["licenses"] = licenses + component["externalReferences"] = [ + _external_reference(label.strip(), url.strip()) + for label, url in (entry.split(",", 1) for entry in metadata.get_all("Project-URL", [])) + ] + if home_page := metadata.get("Home-page"): + component["externalReferences"].insert(0, {"type": "website", "url": home_page}) + return component + +def _licenses(metadata: PackageMetadata) -> list[dict[str, Any]]: + if expression := metadata.get("License-Expression"): + return [{"expression": expression, "acknowledgment": "declared"}] + names = [ + entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") + ] + if (declared := metadata.get("License")) and "\n" not in declared: + names.append(declared) + return [{"license": {"name": name, "acknowledgment": "declared"}} for name in names] -def _bundled_wheels() -> dict[str, str]: - tree = ast.parse(_EMBED_INIT.read_text(encoding="utf-8")) - sha256_by_name = {} + +def _bundle_support() -> dict[str, dict[str, str]]: + tree = ast.parse((_EMBED / "__init__.py").read_text(encoding="utf-8")) for node in tree.body: if isinstance(node, ast.Assign) and any( - isinstance(target, ast.Name) and target.id == "BUNDLE_SHA256" for target in node.targets + isinstance(target, ast.Name) and target.id == "BUNDLE_SUPPORT" for target in node.targets ): - sha256_by_name = ast.literal_eval(node.value) - break - if not sha256_by_name: - msg = f"BUNDLE_SHA256 not found in {_EMBED_INIT}" - raise RuntimeError(msg) - return sha256_by_name + return ast.literal_eval(node.value) + msg = f"BUNDLE_SUPPORT not found in {_EMBED / '__init__.py'}" + raise RuntimeError(msg) + + +def _declared_dependency(requirement: str) -> dict[str, Any]: + parsed = Requirement(requirement) + component = { + "type": "library", + # the same distribution can be declared more than once with different markers, so the purl alone is not unique + "bom-ref": f"requires-dist:{requirement}", + "name": parsed.name, + "purl": _purl(parsed.name), + "externalReferences": [{"type": "distribution", "url": f"https://pypi.org/project/{parsed.name}/"}], + "properties": [{"name": "virtualenv:requires-dist", "value": requirement}], + } + if parsed.marker is not None: + component["properties"].append({"name": "virtualenv:environment-marker", "value": str(parsed.marker)}) + return component -def _serial_number(name: str, version: str, wheel_sha256: dict[str, str]) -> str: - # uuid5 rather than uuid4: deterministic on the inputs that actually change the SBOM's content, so two builds - # of the same commit against the same bundled wheels produce a byte-identical document - payload = f"{name}@{version}+{','.join(f'{k}:{v}' for k, v in sorted(wheel_sha256.items()))}" - return f"urn:uuid:{uuid.uuid5(_SBOM_NAMESPACE, payload)}" +def _build_tools(package_version: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + hook = Path(__file__) + interpreter = f"pkg:generic/{sys.implementation.name}@{platform.python_version()}" + tools = [ + { + "type": "application", + "bom-ref": "tool:hatch_build.py", + "name": hook.name, + "version": package_version, + "hashes": [{"alg": "SHA-256", "content": hashlib.sha256(hook.read_bytes()).hexdigest()}], + "externalReferences": [{"type": "vcs", "url": f"{_REPOSITORY}/blob/main/hatch_build.py"}], + }, + { + "type": "platform", + "bom-ref": f"tool:{interpreter}", + "name": sys.implementation.name, + "version": platform.python_version(), + "description": sys.version, + "purl": interpreter, + }, + { + "type": "operating-system", + "bom-ref": f"tool:os:{platform.system()}@{platform.release()}", + "name": platform.system(), + "version": platform.release(), + "description": platform.platform(), + "properties": [{"name": "machine", "value": platform.machine()}], + }, + ] + # bom-refs are prefixed because the same distribution can be both a build tool and a bundled component + installed = {_purl(distribution.metadata["Name"]): distribution for distribution in distributions()} + tool_dependencies = [] + for distribution in (installed[key] for key in sorted(installed)): + component = _component_from_metadata(distribution.metadata, "library") + component["bom-ref"] = f"tool:{component['purl']}" + tools.append(component) + tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) + return tools, tool_dependencies -def _timestamp() -> str: - # honors SOURCE_DATE_EPOCH through hatchling's own helper, the same value it uses for the wheel's zip entry - # timestamps, so setting it for a reproducible build keeps the SBOM byte-identical too - return datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).isoformat() +def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: + refs = set() + for requirement in map(Requirement, distribution.requires or []): + if (requirement.marker is None or requirement.marker.evaluate()) and ( + dependency := installed.get(_purl(requirement.name)) + ): + refs.add(f"tool:{_purl(dependency.metadata['Name'], dependency.version)}") + return sorted(refs) + + +def _workflow(root: dict[str, Any], tools: list[dict[str, Any]]) -> dict[str, Any]: + workflow: dict[str, Any] = { + "bom-ref": "workflow:wheel-build", + "uid": "wheel-build", + "name": "build the wheel and this SBOM", + "taskTypes": ["build"], + "resourceReferences": [{"ref": tool["bom-ref"]} for tool in tools], + "outputs": [{"type": "artifact", "resource": {"ref": root["bom-ref"]}}], + } + if recorded := [ + {"name": name, "value": os.environ[name]} + for name in ("SOURCE_DATE_EPOCH", *_GITHUB_PROVENANCE) + if name in os.environ + ]: + workflow["inputs"] = [{"environmentVars": recorded}] + return workflow diff --git a/pyproject.toml b/pyproject.toml index bb30c5860..964eaf64e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -153,6 +153,7 @@ lint = [ ] pkg-meta = [ "check-wheel-contents>=0.6.3", + "cyclonedx-python-lib[json-validation]>=11.12", "twine>=6.2", "uv>=0.10.2", ] diff --git a/tasks/validate_sbom.py b/tasks/validate_sbom.py index 95efa5299..4dc38ed56 100644 --- a/tasks/validate_sbom.py +++ b/tasks/validate_sbom.py @@ -1,4 +1,4 @@ -"""Check that a built wheel's embedded SBOM satisfies what a consumer like actions/attest requires.""" +"""Check that a built wheel's embedded SBOM is valid CycloneDX 1.6 and satisfies what actions/attest requires.""" from __future__ import annotations @@ -9,6 +9,9 @@ from pathlib import Path from typing import Any +from cyclonedx.schema import SchemaVersion +from cyclonedx.validation.json import JsonStrictValidator + _SERIAL_PATTERN = re.compile(r"^urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") @@ -24,64 +27,52 @@ def main() -> None: for problem in problems: print(f"SBOM invalid: {problem}") # ruff:ignore[print] raise SystemExit(1) - print(f"SBOM in {wheel.name} is structurally valid") # ruff:ignore[print] + print(f"SBOM in {wheel.name} is valid") # ruff:ignore[print] def validate(wheel: Path) -> list[str]: with zipfile.ZipFile(wheel) as archive: - names = archive.namelist() - sbom_name = next((name for name in names if name.endswith("/sboms/virtualenv.cdx.json")), None) + sbom_name = next((n for n in archive.namelist() if n.endswith("/sboms/virtualenv.cdx.json")), None) if sbom_name is None: return ["no .dist-info/sboms/virtualenv.cdx.json found in the wheel"] - document = json.loads(archive.read(sbom_name)) + raw = archive.read(sbom_name).decode("utf-8") + + if schema_error := JsonStrictValidator(SchemaVersion.V1_6).validate_str(raw): + return [f"does not conform to the CycloneDX 1.6 schema: {schema_error}"] + document = json.loads(raw) problems = [] - if document.get("bomFormat") != "CycloneDX": - problems.append(f"bomFormat must be 'CycloneDX', got {document.get('bomFormat')!r}") - if document.get("specVersion") != "1.6": - problems.append(f"specVersion must be '1.6', got {document.get('specVersion')!r}") - if document.get("version") != 1: - problems.append(f"version must be 1, got {document.get('version')!r}") - - serial = document.get("serialNumber") - if not serial or not _SERIAL_PATTERN.match(serial): + if not _SERIAL_PATTERN.match(document.get("serialNumber", "")): # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize # the document as CycloneDX at all, and silently rejects anything missing it as an unknown format - problems.append(f"serialNumber must match {_SERIAL_PATTERN.pattern}, got {serial!r}") - - metadata_problems, root_ref = _validate_metadata(document.get("metadata", {})) - problems += metadata_problems - problems += _validate_dependency_graph(document, root_ref) + problems.append(f"serialNumber must match {_SERIAL_PATTERN.pattern}, got {document.get('serialNumber')!r}") + problems += _validate_references(document) return problems -def _validate_metadata(metadata: dict[str, Any]) -> tuple[list[str], str | None]: - problems = [] - if not metadata.get("timestamp"): - problems.append("metadata.timestamp is missing") - if not metadata.get("tools", {}).get("components"): - problems.append("metadata.tools.components is missing or empty") - root_ref = metadata.get("component", {}).get("bom-ref") - if not root_ref: - problems.append("metadata.component.bom-ref is missing") - return problems, root_ref - +def _validate_references(document: dict[str, Any]) -> list[str]: + root_ref = document["metadata"]["component"]["bom-ref"] + component_refs = {component["bom-ref"] for component in document["components"]} + tool_refs = {tool["bom-ref"] for tool in document["metadata"]["tools"]["components"]} + known = {root_ref, *component_refs, *tool_refs} + if len(known) != 1 + len(document["components"]) + len(tool_refs): + return ["bom-ref values are not unique across the root, components and tools"] -def _validate_dependency_graph(document: dict[str, Any], root_ref: str | None) -> list[str]: problems = [] - component_refs = {component.get("bom-ref") for component in document.get("components", [])} - dependency_entries = { - entry.get("ref"): set(entry.get("dependsOn", [])) for entry in document.get("dependencies", []) - } - unknown_refs = set(dependency_entries) - component_refs - {root_ref} - if unknown_refs: - problems.append(f"dependencies entries with no matching component: {sorted(unknown_refs)}") - - root_depends_on = dependency_entries.get(root_ref) - if root_depends_on is None: - problems.append(f"no dependencies entry for the root component {root_ref!r}") - elif root_depends_on != component_refs: - problems.append(f"root dependsOn {sorted(root_depends_on)} does not match components {sorted(component_refs)}") + dependencies = {entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]} + if unknown := (set(dependencies) | set().union(*dependencies.values())) - known: + problems.append(f"dependencies reference bom-refs that do not exist: {sorted(unknown)}") + if dependencies.get(root_ref) != component_refs: + problems.append( + f"root dependsOn {sorted(dependencies.get(root_ref, []))} != components {sorted(component_refs)}" + ) + if missing := component_refs - set(dependencies): + problems.append(f"components without a dependencies entry: {sorted(missing)}") + + for workflow in document["formulation"][0]["workflows"]: + referenced = {reference["ref"] for reference in workflow["resourceReferences"]} + if unknown := referenced - tool_refs: + problems.append(f"workflow {workflow['uid']} references unknown tools: {sorted(unknown)}") return problems From 2a58c7376775d517b69a91604b8ad14625665b8f Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 19 Sep 2026 09:42:19 -0700 Subject: [PATCH 2/4] =?UTF-8?q?=F0=9F=93=9D=20docs(changelog):=20add=20fra?= =?UTF-8?q?gments=20for=20#3270?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog/3270.bugfix.rst | 2 ++ docs/changelog/3270.feature.rst | 4 ++++ 2 files changed, 6 insertions(+) create mode 100644 docs/changelog/3270.bugfix.rst create mode 100644 docs/changelog/3270.feature.rst diff --git a/docs/changelog/3270.bugfix.rst b/docs/changelog/3270.bugfix.rst new file mode 100644 index 000000000..06a34bb23 --- /dev/null +++ b/docs/changelog/3270.bugfix.rst @@ -0,0 +1,2 @@ +Fix the embedded SBOM not conforming to the CycloneDX 1.6 schema (an invalid ``vendor`` key on the generator tool entry) +and validate it against the schema in CI. diff --git a/docs/changelog/3270.feature.rst b/docs/changelog/3270.feature.rst new file mode 100644 index 000000000..854012601 --- /dev/null +++ b/docs/changelog/3270.feature.rst @@ -0,0 +1,4 @@ +The embedded SBOM now describes the root component's license, copyright, maintainers and project links, each bundled +wheel from its own metadata, the declared runtime dependencies, and the full build environment (interpreter, OS and +every distribution in the isolated build environment with the dependency graph between them), plus the GitHub Actions +run when built there. From 0322cc2f1b590d858103c29b8f41d1d8111f9191 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 19 Sep 2026 09:43:06 -0700 Subject: [PATCH 3/4] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(tasks):=20spe?= =?UTF-8?q?ll=20out=20the=20UUID=20serial=20regex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Named groups for the RFC 4122 fields read better than a run of hex quantifiers, and the error message no longer prints the raw pattern. --- tasks/validate_sbom.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tasks/validate_sbom.py b/tasks/validate_sbom.py index 4dc38ed56..ca6a60a06 100644 --- a/tasks/validate_sbom.py +++ b/tasks/validate_sbom.py @@ -7,12 +7,23 @@ import sys import zipfile from pathlib import Path -from typing import Any +from typing import Any, Final from cyclonedx.schema import SchemaVersion from cyclonedx.validation.json import JsonStrictValidator -_SERIAL_PATTERN = re.compile(r"^urn:uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") +_SERIAL_PATTERN: Final[re.Pattern[str]] = re.compile( + r""" + ^urn:uuid: + (?P[0-9a-f]{8})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{4})- + (?P[0-9a-f]{12}) + $ + """, + re.VERBOSE, +) def main() -> None: @@ -45,7 +56,7 @@ def validate(wheel: Path) -> list[str]: if not _SERIAL_PATTERN.match(document.get("serialNumber", "")): # optional in the CycloneDX spec itself, but actions/attest's format sniffer requires it to recognize # the document as CycloneDX at all, and silently rejects anything missing it as an unknown format - problems.append(f"serialNumber must match {_SERIAL_PATTERN.pattern}, got {document.get('serialNumber')!r}") + problems.append(f"serialNumber must be a urn:uuid: RFC 4122 UUID, got {document.get('serialNumber')!r}") problems += _validate_references(document) return problems From b26ef0645f0b3fb90640d281586e13622450d12a Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 19 Sep 2026 09:51:34 -0700 Subject: [PATCH 4/4] =?UTF-8?q?=E2=9C=A8=20feat(build):=20record=20every?= =?UTF-8?q?=20file,=20revision=20and=20license=20in=20the=20SBOM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit codespell had rewritten the CycloneDX `acknowledgement` key to American spelling after the last schema validation ran, which the schema rejects since license objects forbid unknown keys. The key is now on codespell's ignore list. The timestamp fell back to hatchling's fixed 2020 value whenever SOURCE_DATE_EPOCH was unset, so released SBOMs claimed a creation date years in the past. The hook now honors SOURCE_DATE_EPOCH and otherwise records the real time, and the release workflow pins SOURCE_DATE_EPOCH to the release commit so release artifacts are reproducible. Every bundled wheel and build tool now lists its files with the RECORD hashes and sizes, and bundled wheels carry identity evidence stating how they were identified. Console-script launchers are excluded from the build tool listings because their shebang embeds the build env path, which made consecutive builds differ. The root component records the exact source revision, the PyPI attestation location, classifiers and keywords; the interpreter records its compiler and build; Linux hosts record os-release; the document declares its own license. --- .github/workflows/release.yaml | 2 + hatch_build.py | 166 +++++++++++++++++++++++++++------ pyproject.toml | 2 +- tasks/validate_sbom.py | 24 ++++- 4 files changed, 163 insertions(+), 31 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index bf4ba4131..904c4cdfb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -27,6 +27,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Set up Python run: uv python install 3.14 + - name: Pin build timestamps to the release commit for reproducible artifacts + run: echo "SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)" >> "$GITHUB_ENV" - name: Build sdist and wheel run: uv build --python 3.14 --python-preference only-managed --sdist --wheel . --out-dir dist - name: Extract the SBOM the wheel build embedded diff --git a/hatch_build.py b/hatch_build.py index 0d9beb7fb..e48bde0a7 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -1,11 +1,14 @@ from __future__ import annotations import ast +import base64 import hashlib import json import os import platform import re +import shutil +import subprocess import sys import tempfile import uuid @@ -19,7 +22,6 @@ from typing import TYPE_CHECKING, Any, Final from hatchling.builders.hooks.plugin.interface import BuildHookInterface -from hatchling.builders.utils import get_reproducible_timestamp from packaging.requirements import Requirement if TYPE_CHECKING: @@ -52,6 +54,21 @@ "github": "vcs", "chat": "chat", } +# core metadata headers copied verbatim onto a component as properties +_METADATA_PROPERTIES: Final[tuple[str, ...]] = ( + "Requires-Python", + "Requires-Dist", + "Provides-Extra", + "Classifier", + "Keywords", +) +_RECORD_HASH_ALGORITHMS: Final[dict[str, str]] = { + "md5": "MD5", + "sha1": "SHA-1", + "sha256": "SHA-256", + "sha384": "SHA-384", + "sha512": "SHA-512", +} # only these names are ever read from the environment: they identify the CI run and carry no secrets _GITHUB_PROVENANCE: Final[tuple[str, ...]] = ( "GITHUB_REPOSITORY", @@ -71,11 +88,12 @@ class SbomBuildHook(BuildHookInterface): General-purpose SBOM scanners (syft, cyclonedx-py, GitHub's dependency graph) read declared dependency metadata or an installed environment, and virtualenv's bundled pip and setuptools wheels are neither: they are data files embedded under ``src/virtualenv/seed/wheels/embed/``, invisible to every one of those tools. Each bundled wheel is - described from its own ``METADATA`` and hashed from its bytes, so a wheel bump needs no separate SBOM update. + described from its own ``METADATA`` and ``RECORD`` and hashed from its bytes, so a wheel bump needs no separate SBOM + update. - The document also records the build environment (interpreter, OS, every distribution in the isolated build env and - the dependency graph between them), which PEP 770 calls out as what a third party needs to verify build - reproducibility, plus the CI run that produced the wheel when built in GitHub Actions. + The document also records the build environment (interpreter, OS, every distribution in the isolated build env with + its files and the dependency graph between them), which PEP 770 calls out as what a third party needs to verify + build reproducibility, plus the source revision and the CI run that produced the wheel when known. """ @@ -102,13 +120,14 @@ def _cyclonedx_document(core: CoreMetadata, version: str) -> dict[str, Any]: tools, tool_dependencies = _build_tools(version) body = { "metadata": { - "timestamp": datetime.fromtimestamp(get_reproducible_timestamp(), tz=timezone.utc).isoformat(), + "timestamp": _timestamp(), "lifecycles": [{"phase": "build"}], "tools": {"components": tools}, "manufacturer": _PYPA, "authors": _contacts(core.maintainers_data["name"], core.maintainers_data["email"]), "supplier": _PYPA, "component": root, + "licenses": [{"expression": core.license_expression, "acknowledgement": "declared"}], "properties": [ { "name": "virtualenv:sbom:scope", @@ -141,15 +160,25 @@ def _cyclonedx_document(core: CoreMetadata, version: str) -> dict[str, Any]: def _root_component(core: CoreMetadata, version: str) -> dict[str, Any]: purl = _purl(core.name, version) + wheel_name = f"{core.name}-{version}-py3-none-any.whl" license_lines = (_ROOT / "LICENSE").read_text(encoding="utf-8").splitlines() references = list(starmap(_external_reference, core.urls.items())) references += [ {"type": "distribution", "url": f"https://pypi.org/project/{core.name}/{version}/"}, + {"type": "attestation", "url": f"https://pypi.org/integrity/{core.name}/{version}/{wheel_name}/provenance"}, {"type": "release-notes", "url": "https://virtualenv.pypa.io/en/latest/changelog.html"}, {"type": "security-contact", "url": f"{_REPOSITORY}/security/policy"}, {"type": "advisories", "url": f"{_REPOSITORY}/security/advisories"}, {"type": "license", "url": f"{_REPOSITORY}/blob/main/LICENSE"}, ] + properties = [ + {"name": "virtualenv:requires-python", "value": core.requires_python}, + *({"name": "python:classifier", "value": classifier} for classifier in core.classifiers), + *({"name": "python:keyword", "value": keyword} for keyword in core.keywords), + ] + if commit := _commit(): + references.append({"type": "vcs", "url": f"{_REPOSITORY}/tree/{commit}", "comment": "exact source revision"}) + properties.append({"name": "virtualenv:vcs-commit", "value": commit}) if run_id := os.environ.get("GITHUB_RUN_ID"): server = os.environ.get("GITHUB_SERVER_URL", "https://github.com") references.append( @@ -163,11 +192,11 @@ def _root_component(core: CoreMetadata, version: str) -> dict[str, Any]: "name": core.name, "version": version, "description": core.description, - "licenses": [{"expression": core.license_expression, "acknowledgment": "declared"}], + "licenses": [{"expression": core.license_expression, "acknowledgement": "declared"}], "copyright": next(line for line in license_lines if line.startswith("Copyright")), "purl": purl, "externalReferences": references, - "properties": [{"name": "virtualenv:requires-python", "value": core.requires_python}], + "properties": properties, } @@ -176,6 +205,21 @@ def _purl(name: str, version: str | None = None) -> str: return f"pkg:pypi/{normalized}@{version}" if version else f"pkg:pypi/{normalized}" +def _external_reference(label: str, url: str) -> dict[str, str]: + reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") + return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} + + +def _commit() -> str | None: + if commit := os.environ.get("GITHUB_SHA"): + return commit + if not (_ROOT / ".git").exists() or (git := shutil.which("git")) is None: # building from an sdist + return None + return subprocess.run( + [git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True, cwd=_ROOT + ).stdout.strip() + + def _contacts(names: list[str], addresses: list[str]) -> list[dict[str, str]]: contacts = [{"name": name} for name in names] for name, email in getaddresses(addresses): @@ -183,19 +227,16 @@ def _contacts(names: list[str], addresses: list[str]) -> list[dict[str, str]]: return contacts -def _external_reference(label: str, url: str) -> dict[str, str]: - reference_type = _URL_LABEL_TO_REFERENCE_TYPE.get(re.sub(r"[^a-z]", "", label.lower()), "other") - return {"type": reference_type, "url": url, "comment": f"Project-URL: {label}"} - - def _bundled_component(wheel: Path) -> dict[str, Any]: with zipfile.ZipFile(wheel) as archive: metadata_name = next(name for name in archive.namelist() if name.endswith(".dist-info/METADATA")) metadata = Parser().parsestr(archive.read(metadata_name).decode("utf-8")) + record = archive.read(metadata_name.replace("METADATA", "RECORD")).decode("utf-8") component = _component_from_metadata(metadata, "library") if any(reference["url"].startswith("https://github.com/pypa/") for reference in component["externalReferences"]): component["supplier"] = _PYPA - component["hashes"] = [{"alg": "SHA-256", "content": hashlib.sha256(wheel.read_bytes()).hexdigest()}] + sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + component["hashes"] = [{"alg": "SHA-256", "content": sha256}] component["externalReferences"].append( {"type": "distribution", "url": f"https://pypi.org/project/{metadata['Name']}/{metadata['Version']}/"}, ) @@ -206,9 +247,27 @@ def _bundled_component(wheel: Path) -> dict[str, Any]: for python_version, wheels in _bundle_support().items() if wheel.name in wheels.values() ), + *component["properties"], + ] + component["evidence"] = { + "identity": [ + { + "field": "purl", + "confidence": 1, + "methods": [{"technique": "manifest-analysis", "confidence": 1, "value": metadata_name}], + }, + { + "field": "hash", + "confidence": 1, + "methods": [{"technique": "hash-comparison", "confidence": 1, "value": sha256}], + }, + ], + } + component["components"] = [ + _file_component(component["bom-ref"], path, digest, size) + for path, digest, size in (line.split(",") for line in record.splitlines() if line) + if digest ] - if requires_python := metadata.get("Requires-Python"): - component["properties"].append({"name": "virtualenv:requires-python", "value": requires_python}) return component @@ -237,18 +296,23 @@ def _component_from_metadata(metadata: PackageMetadata, component_type: str) -> ] if home_page := metadata.get("Home-page"): component["externalReferences"].insert(0, {"type": "website", "url": home_page}) + component["properties"] = [ + {"name": f"python:{header.lower()}", "value": value} + for header in _METADATA_PROPERTIES + for value in metadata.get_all(header, []) + ] return component def _licenses(metadata: PackageMetadata) -> list[dict[str, Any]]: if expression := metadata.get("License-Expression"): - return [{"expression": expression, "acknowledgment": "declared"}] + return [{"expression": expression, "acknowledgement": "declared"}] names = [ entry.rsplit(" :: ", 1)[-1] for entry in metadata.get_all("Classifier", []) if entry.startswith("License :: ") ] if (declared := metadata.get("License")) and "\n" not in declared: names.append(declared) - return [{"license": {"name": name, "acknowledgment": "declared"}} for name in names] + return [{"license": {"name": name, "acknowledgement": "declared"}} for name in names] def _bundle_support() -> dict[str, dict[str, str]]: @@ -262,6 +326,19 @@ def _bundle_support() -> dict[str, dict[str, str]]: raise RuntimeError(msg) +def _file_component(parent_ref: str, path: str, digest: str, size: str) -> dict[str, Any]: + # RECORD stores "=", CycloneDX wants lowercase hex + algorithm, _, encoded = digest.partition("=") + raw = base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4)) + return { + "type": "file", + "bom-ref": f"{parent_ref}#{path}", + "name": path, + "hashes": [{"alg": _RECORD_HASH_ALGORITHMS[algorithm], "content": raw.hex()}], + "properties": [{"name": "size", "value": size}], + } + + def _declared_dependency(requirement: str) -> dict[str, Any]: parsed = Requirement(requirement) component = { @@ -297,15 +374,13 @@ def _build_tools(package_version: str) -> tuple[list[dict[str, Any]], list[dict[ "version": platform.python_version(), "description": sys.version, "purl": interpreter, + "properties": [ + {"name": "python:implementation", "value": platform.python_implementation()}, + {"name": "python:compiler", "value": platform.python_compiler()}, + {"name": "python:build", "value": " ".join(platform.python_build())}, + ], }, - { - "type": "operating-system", - "bom-ref": f"tool:os:{platform.system()}@{platform.release()}", - "name": platform.system(), - "version": platform.release(), - "description": platform.platform(), - "properties": [{"name": "machine", "value": platform.machine()}], - }, + _operating_system(), ] # bom-refs are prefixed because the same distribution can be both a build tool and a bundled component installed = {_purl(distribution.metadata["Name"]): distribution for distribution in distributions()} @@ -313,11 +388,42 @@ def _build_tools(package_version: str) -> tuple[list[dict[str, Any]], list[dict[ for distribution in (installed[key] for key in sorted(installed)): component = _component_from_metadata(distribution.metadata, "library") component["bom-ref"] = f"tool:{component['purl']}" + component["components"] = [ + _file_component( + component["bom-ref"], file.as_posix(), f"{file.hash.mode}={file.hash.value}", str(file.size) + ) + for file in distribution.files or [] + # console-script launchers live outside site-packages and embed the build env's interpreter path in + # their shebang, so their hash differs on every build and says nothing about the distribution + if file.hash is not None and not file.as_posix().startswith("../") + ] tools.append(component) tool_dependencies.append({"ref": component["bom-ref"], "dependsOn": _depends_on(distribution, installed)}) return tools, tool_dependencies +def _operating_system() -> dict[str, Any]: + component: dict[str, Any] = { + "type": "operating-system", + "bom-ref": f"tool:os:{platform.system()}@{platform.release()}", + "name": platform.system(), + "version": platform.release(), + "description": platform.platform(), + "properties": [ + {"name": "machine", "value": platform.machine()}, + {"name": "kernel-version", "value": platform.version()}, + ], + } + try: + os_release = platform.freedesktop_os_release() + except OSError: # not a freedesktop system, e.g. macOS or Windows + return component + component["properties"] += [ + {"name": f"os-release:{key}", "value": value} for key, value in sorted(os_release.items()) + ] + return component + + def _depends_on(distribution: Distribution, installed: dict[str, Distribution]) -> list[str]: refs = set() for requirement in map(Requirement, distribution.requires or []): @@ -344,3 +450,11 @@ def _workflow(root: dict[str, Any], tools: list[dict[str, Any]]) -> dict[str, An ]: workflow["inputs"] = [{"environmentVars": recorded}] return workflow + + +def _timestamp() -> str: + # SOURCE_DATE_EPOCH makes the document reproducible; without it the real creation time is the truthful value, + # not hatchling's fixed 2020 fallback + if (epoch := os.environ.get("SOURCE_DATE_EPOCH")) is not None: + return datetime.fromtimestamp(int(epoch), tz=timezone.utc).isoformat() + return datetime.now(tz=timezone.utc).replace(microsecond=0).isoformat() diff --git a/pyproject.toml b/pyproject.toml index 964eaf64e..9611cd92d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -236,7 +236,7 @@ lint.preview = true [tool.codespell] builtin = "clear,usage,en-GB_to_en-US" -ignore-words-list = "intoto" +ignore-words-list = "acknowledgement,intoto" count = true [tool.pyproject-fmt] diff --git a/tasks/validate_sbom.py b/tasks/validate_sbom.py index ca6a60a06..786eaac53 100644 --- a/tasks/validate_sbom.py +++ b/tasks/validate_sbom.py @@ -7,11 +7,14 @@ import sys import zipfile from pathlib import Path -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final from cyclonedx.schema import SchemaVersion from cyclonedx.validation.json import JsonStrictValidator +if TYPE_CHECKING: + from collections.abc import Iterator + _SERIAL_PATTERN: Final[re.Pattern[str]] = re.compile( r""" ^urn:uuid: @@ -65,9 +68,16 @@ def _validate_references(document: dict[str, Any]) -> list[str]: root_ref = document["metadata"]["component"]["bom-ref"] component_refs = {component["bom-ref"] for component in document["components"]} tool_refs = {tool["bom-ref"] for tool in document["metadata"]["tools"]["components"]} - known = {root_ref, *component_refs, *tool_refs} - if len(known) != 1 + len(document["components"]) + len(tool_refs): - return ["bom-ref values are not unique across the root, components and tools"] + every_ref = list( + _bom_refs([ + document["metadata"]["component"], + *document["components"], + *document["metadata"]["tools"]["components"], + ]) + ) + if duplicates := {ref for ref in every_ref if every_ref.count(ref) > 1}: + return [f"bom-ref values are not unique: {sorted(duplicates)}"] + known = set(every_ref) problems = [] dependencies = {entry["ref"]: set(entry.get("dependsOn", [])) for entry in document["dependencies"]} @@ -87,5 +97,11 @@ def _validate_references(document: dict[str, Any]) -> list[str]: return problems +def _bom_refs(components: list[dict[str, Any]]) -> Iterator[str]: + for component in components: + yield component["bom-ref"] + yield from _bom_refs(component.get("components", [])) + + if __name__ == "__main__": main()