From a599b56a3f5d4af51ce1bde655d36755c6a0e7db Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 15:25:24 +0000 Subject: [PATCH 01/40] feat(sbom): add bounded software lineage ingestion (#995) --- docs/SBOM.md | 43 + perseus.py | 1633 +++++++++++++++++++++++- scripts/build.py | 1 + src/perseus/cli.py | 22 + src/perseus/sbom_lineage.py | 1610 +++++++++++++++++++++++ tests/fixtures/sbom/cyclonedx-app.json | 40 + tests/fixtures/sbom/cyclonedx-app.xml | 26 + tests/fixtures/sbom/spdx-app.json | 57 + tests/fixtures/sbom/spdx-app.xml | 28 + tests/fixtures/sbom/spdx-rdf.xml | 39 + tests/test_sbom_lineage.py | 353 +++++ 11 files changed, 3851 insertions(+), 1 deletion(-) create mode 100644 src/perseus/sbom_lineage.py create mode 100644 tests/fixtures/sbom/cyclonedx-app.json create mode 100644 tests/fixtures/sbom/cyclonedx-app.xml create mode 100644 tests/fixtures/sbom/spdx-app.json create mode 100644 tests/fixtures/sbom/spdx-app.xml create mode 100644 tests/fixtures/sbom/spdx-rdf.xml create mode 100644 tests/test_sbom_lineage.py diff --git a/docs/SBOM.md b/docs/SBOM.md index e8d26da8..36a81b1c 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -95,3 +95,46 @@ - [x] Dependency relationship: listed above - [x] SBOM author: Perseus Computing LLC - [x] Timestamp: included + +--- + +## Queryable SBOM and lineage contract (#995) + +Perseus also provides an offline, stdlib-only normalization and query surface +for SBOMs produced by an existing scanner or build pipeline. It does not replace +those tools and it does not infer a clean result from an incomplete document. + +Supported input formats: + +- SPDX 2.2/2.3 JSON and XML; +- CycloneDX 1.4/1.5/1.6 JSON and XML. + +Every normalized document records its format/version, source reference, raw +document SHA-256, supplier/timestamp metadata when supplied, component and +relationship counts, and an ingestion digest. Component projections retain +names, versions, package identifiers, licenses, and supplied vulnerability, +VEX, signature, attestation, advisory, or documentation references. Missing +metadata is represented as `partial` coverage with explicit `unknown` fields. + +A local graph can add pipeline-owned edges for: + +```text +source -> dependency -> build -> artifact -> deployment +``` + +Each edge carries explicit confidence (`high`, `medium`, `low`, or `unknown`), +coverage (`complete`, `partial`, or `unknown`), and optional evidence +references. The impacted-artifact query returns the traversed path and evidence +references. A query with incomplete coverage returns `unknown` or `partial`; +`not_affected` is never asserted merely because no artifact was found. + +Example offline commands: + +```bash +perseus sbom ingest build.spdx.json --output normalized.json +perseus sbom merge normalized.json --edges pipeline-edges.json --output lineage.json +perseus sbom query lineage.json CVE-2021-44228 --json +``` + +The core path requires no cloud service. Deterministic JSON/XML fixtures and +contract tests live under `tests/fixtures/sbom/` and `tests/test_sbom_lineage.py`. diff --git a/perseus.py b/perseus.py index 57e4048d..9167021e 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "cdfc6e2-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "da2025e-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44435,6 +44435,1615 @@ def cmd_code_map(args, cfg) -> int: symbols = ", ".join(symbol["name"] for symbol in item.get("symbols", [])) or "(no symbols)" print(f" {item['candidate_id']} {symbols} [{item['selection_reason']}]") return 0 +"""Offline SPDX/CycloneDX ingestion and queryable software lineage (#995). + +The module deliberately stays stdlib-only. It normalizes the two common SBOM +families into a bounded, digest-sealed projection and keeps coverage/unknown +states visible. It does not replace a generator, scanner, VEX authority, or +signing system: references are carried through only when the input supplies +those references. +""" + +import hashlib +import json +import re +import xml.etree.ElementTree as _sl_et +from collections import deque +from pathlib import Path +from typing import Any, Mapping + +_SL_SCHEMA = "perseus-sbom/v1" +_SL_LINEAGE_SCHEMA = "perseus-software-lineage/v1" +_SL_QUERY_SCHEMA = "perseus-software-lineage-query/v1" +_SL_FORMATS = frozenset({"SPDX", "CycloneDX"}) +_SL_SPDX_VERSIONS = frozenset({"2.2", "2.3"}) +_SL_CDX_VERSIONS = frozenset({"1.4", "1.5", "1.6"}) +_SL_CONFIDENCE = frozenset({"high", "medium", "low", "unknown"}) +_SL_COVERAGE = frozenset({"complete", "partial", "unknown"}) +_SL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") +_SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") +_SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") +_SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") +_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer\s+|basic\s+|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*=|credential\s*=)") +_SL_REFERENCE_TYPES = frozenset({ + "advisory", "attestation", "cve", "distribution", "documentation", "license", + "purl", "signature", "vex", "vulnerability", "website", "other", +}) +_SL_FORBIDDEN_MARKERS = frozenset({"api_key", "authorization", "password", "secret", "token", "credential"}) +_SL_MAX_INPUT_BYTES = 8 * 1024 * 1024 +_SL_MAX_XML_ELEMENTS = 20_000 +_SL_MAX_XML_DEPTH = 128 +_SL_MAX_COMPONENTS = 512 +_SL_MAX_RELATIONSHIPS = 1024 +_SL_MAX_DOCUMENTS = 64 +_SL_MAX_EDGES = 4096 +_SL_MAX_REFERENCES = 64 +_SL_MAX_PROPERTIES = 64 +_SL_MAX_IDENTIFIERS = 64 +_SL_MAX_LICENSES = 32 +_SL_MAX_HASHES = 8 +_SL_MAX_EVIDENCE_REFS = 64 +_SL_MAX_DEPENDENCY_TARGETS = 128 +_SL_MAX_QUERY_MATCHES = 256 +_SL_MAX_QUERY_RESULTS = 256 +_SL_MAX_PATH_LENGTH = 32 +_SL_MAX_NODES = 20_000 + + +class SBOMLineageError(ValueError): + """Raised when an SBOM or lineage projection cannot be verified.""" + + +def _sl_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +def _sl_sha(value: Any) -> str: + return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() + + +def _sl_sensitive(value: str) -> bool: + """Return whether a scalar looks like it carries a credential.""" + authority = value.split("://", 1)[1].split("/", 1)[0] if "://" in value else "" + return bool(_SL_SENSITIVE_REFERENCE_RE.search(value) or (authority and "@" in authority)) + + +def _sl_truncated(truncated: list[str] | None, name: str) -> None: + if truncated is not None and name not in truncated: + truncated.append(name) + + +def _sl_list(value: Any, field: str) -> list[Any]: + if value is None: + return [] + if not isinstance(value, list): + raise SBOMLineageError(f"{field} must be a list") + return value + + +def _sl_nonnegative_int(value: Any, field: str, *, maximum: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise SBOMLineageError(f"{field} must be an integer") + if value < 0 or (maximum is not None and value > maximum): + bound = f" between 0 and {maximum}" if maximum is not None else "" + raise SBOMLineageError(f"{field} must be{bound}") + return value + + +def _sl_limit(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= _SL_MAX_QUERY_RESULTS: + raise SBOMLineageError(f"{field} must be between 1 and {_SL_MAX_QUERY_RESULTS}") + return value + + +def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512) -> str: + if value is None: + if required: + raise SBOMLineageError(f"{field} is required") + return "" + if not isinstance(value, str): + value = str(value) + text = re.sub(r"[\x00-\x1f\x7f]", " ", value).strip() + if required and not text: + raise SBOMLineageError(f"{field} is required") + if len(text) > limit: + raise SBOMLineageError(f"{field} exceeds {limit} characters") + return text + + +def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: + text = _sl_text(value if value is not None else fallback, field, required=True, limit=256) + if not _SL_ID_RE.fullmatch(text): + raise SBOMLineageError(f"{field} is not a bounded identifier") + return text + + +def _sl_safe_locator(value: Any, field: str) -> str: + """Keep public identifiers; hash credential-bearing scalar values.""" + text = _sl_text(value, field, required=True, limit=1024) + if _sl_sensitive(text): + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + return text + + +def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: + if value in (None, ""): + return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" + text = _sl_text(value, "source_ref", required=True, limit=512) + if text.startswith("sha256:") and re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text): + return text.lower() + if _SL_PUBLIC_SOURCE_RE.fullmatch(text) and not _SL_SENSITIVE_REFERENCE_RE.search(text): + return text + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sl_strict_text(value: Any, field: str, *, allow_none: bool = False, limit: int = 512) -> str | None: + if value is None and allow_none: + return None + if not isinstance(value, str): + raise SBOMLineageError(f"{field} must be a string") + if value != value.strip() or re.search(r"[\x00-\x1f\x7f]", value): + raise SBOMLineageError(f"{field} contains invalid characters") + if not value or len(value) > limit: + raise SBOMLineageError(f"{field} is empty or exceeds {limit} characters") + if _sl_sensitive(value): + raise SBOMLineageError(f"{field} contains unsafe credential material") + return value + + +def _sl_strict_safe_locator(value: Any, field: str, *, limit: int = 1024) -> str: + text = _sl_strict_text(value, field, limit=limit) + assert text is not None + if _sl_safe_locator(text, field) != text: + raise SBOMLineageError(f"{field} contains unsafe credential material") + return text + + +def _sl_strict_source_ref(value: Any) -> str: + text = _sl_strict_text(value, "source_ref", limit=512) + assert text is not None + if text.startswith("sha256:"): + if not re.fullmatch(r"sha256:[0-9a-f]{64}", text): + raise SBOMLineageError("source_ref digest is malformed") + return text + if not _SL_PUBLIC_SOURCE_RE.fullmatch(text) or _sl_sensitive(text): + raise SBOMLineageError("source_ref is not a visibility-safe reference") + return text + + +def _sl_safe_hashes(value: Any, *, truncated: list[str] | None = None) -> list[dict[str, str]]: + if value is None: + return [] + hashes = _sl_list(value, "hashes") + if len(hashes) > _SL_MAX_HASHES: + _sl_truncated(truncated, "hashes") + result: list[dict[str, str]] = [] + for raw in hashes[:_SL_MAX_HASHES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("hashes must contain objects") + algorithm = _sl_text(raw.get("alg"), "hash_algorithm", required=True, limit=32) + content = _sl_text(raw.get("content"), "hash", required=True, limit=256) + # Only retain actual digest-looking values. Arbitrary hash content is + # an attacker-controlled data channel and is deliberately dropped. + if re.fullmatch(r"[0-9a-fA-F]{32,128}", content): + result.append({"alg": algorithm, "content": content.lower()}) + return result + + +def _sl_local(element: Any) -> str: + return str(getattr(element, "tag", "")).rsplit("}", 1)[-1] + + +def _sl_child(element: Any, name: str) -> Any | None: + for child in list(element): + if _sl_local(child) == name: + return child + return None + + +def _sl_children(element: Any, name: str) -> list[Any]: + return [child for child in list(element) if _sl_local(child) == name] + + +def _sl_xml_text(element: Any, name: str, *, default: str = "") -> str: + if element is None: + return default + child = _sl_child(element, name) + return (child.text or "").strip() if child is not None and child.text else default + + +def _sl_validate_xml_tree(root: Any) -> None: + count = 0 + stack = [(root, 1)] + while stack: + element, depth = stack.pop() + count += 1 + if count > _SL_MAX_XML_ELEMENTS: + raise SBOMLineageError("SBOM XML contains too many elements") + if depth > _SL_MAX_XML_DEPTH: + raise SBOMLineageError("SBOM XML nesting is too deep") + stack.extend((child, depth + 1) for child in list(element)) + + +def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: + wanted = {name.casefold() for name in names} + for child in list(element) if element is not None else []: + if _sl_local(child).casefold() not in wanted: + continue + text = (child.text or "").strip() if child.text else "" + if text: + return text + for key, value in child.attrib.items(): + if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: + return str(value).lstrip("#") + return default + + +def _sl_descendants(root: Any, *names: str) -> list[Any]: + wanted = {name.casefold() for name in names} + return [element for element in root.iter() if _sl_local(element).casefold() in wanted] + + +def _sl_supplier(value: Any) -> str: + if isinstance(value, Mapping): + return _sl_text(value.get("name"), "supplier") + if isinstance(value, list): + names = [_sl_supplier(item) for item in value] + return "; ".join(item for item in names if item) + return _sl_text(value, "supplier") + + +def _sl_normalize_reference(reference_type: Any, locator: Any, *, category: Any = "", comment: Any = None, hashes: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: + ref_type = _sl_text(reference_type, "reference_type", required=True, limit=64).casefold() + locator_text = _sl_safe_locator(locator, "reference_locator") + result: dict[str, Any] = { + "type": ref_type if ref_type in _SL_REFERENCE_TYPES else "other", + "locator": locator_text, + } + category_text = _sl_text(category, "reference_category", limit=64) + if category_text: + result["category"] = category_text + safe_hashes = _sl_safe_hashes(hashes, truncated=truncated) + if safe_hashes: + result["hashes"] = safe_hashes + return result + + +def _sl_spdx_references(value: Any, *, truncated: list[str] | None = None) -> tuple[list[dict[str, Any]], list[str]]: + references: list[dict[str, Any]] = [] + identifiers: list[str] = [] + if value is None: + return references, identifiers + refs = _sl_list(value, "SPDX externalRefs") + if len(refs) > _SL_MAX_REFERENCES: + _sl_truncated(truncated, "externalRefs") + for raw in refs[:_SL_MAX_REFERENCES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("SPDX externalRefs must contain objects") + locator = raw.get("referenceLocator") + ref_type = raw.get("referenceType", "other") + if locator is None: + raise SBOMLineageError("SPDX externalRef requires referenceLocator") + item = _sl_normalize_reference( + ref_type, locator, category=raw.get("referenceCategory"), + hashes=raw.get("hashes"), truncated=truncated, + ) + references.append(item) + if str(ref_type).casefold() == "purl": + identifiers.append(item["locator"]) + return references, identifiers + + +def _sl_cdx_references(value: Any, *, truncated: list[str] | None = None) -> tuple[list[dict[str, Any]], list[str]]: + references: list[dict[str, Any]] = [] + identifiers: list[str] = [] + if value is None: + return references, identifiers + refs = _sl_list(value, "CycloneDX externalReferences") + if len(refs) > _SL_MAX_REFERENCES: + _sl_truncated(truncated, "externalReferences") + for raw in refs[:_SL_MAX_REFERENCES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("CycloneDX externalReferences must contain objects") + locator = raw.get("url") + if not locator: + raise SBOMLineageError("CycloneDX externalReference requires url") + if "hashes" in raw and raw.get("hashes") is not None and not isinstance(raw.get("hashes"), list): + raise SBOMLineageError("CycloneDX externalReference hashes must be a list") + item = _sl_normalize_reference( + raw.get("type", "other"), locator, hashes=raw.get("hashes"), truncated=truncated, + ) + references.append(item) + if str(raw.get("type", "")).casefold() == "purl": + identifiers.append(item["locator"]) + return references, identifiers + + +def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + if value is None: + return result + properties = _sl_list(value, "properties") + if len(properties) > _SL_MAX_PROPERTIES: + _sl_truncated(truncated, "properties") + for raw in properties[:_SL_MAX_PROPERTIES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("properties must contain objects") + name = _sl_text(raw.get("name"), "property_name", limit=128) + prop_value = _sl_text(raw.get("value"), "property_value", required=True, limit=1024) + if name and prop_value and any(marker in name.casefold() for marker in ("vex", "vuln", "signature", "attestation")): + result.append({ + "type": name, + "locator": "sha256:" + hashlib.sha256(prop_value.encode("utf-8")).hexdigest(), + }) + return result + + +def _sl_component( + *, + component_id: Any, + name: Any, + version: Any = None, + supplier: Any = None, + identifiers: Any = None, + references: Any = None, + component_type: Any = None, + licenses: Any = None, + truncated: list[str] | None = None, +) -> dict[str, Any]: + normalized_name = _sl_text(name, "component_name", required=True, limit=256) + normalized_version = _sl_text(version, "component_version", limit=128) + normalized_id = _sl_id(component_id, "component ID") + ids = {normalized_id} + component_truncated: list[str] = list(truncated or []) + if identifiers is not None and not isinstance(identifiers, (list, tuple, set)): + raise SBOMLineageError("component identifiers must be a list") + identifier_values = list(identifiers or []) + if len(identifier_values) > _SL_MAX_IDENTIFIERS: + _sl_truncated(component_truncated, "identifiers") + for identifier in identifier_values[:_SL_MAX_IDENTIFIERS]: + text = _sl_safe_locator(identifier, "component_identifier") + if text: + ids.add(text) + safe_references = [] + if references is not None and not isinstance(references, (list, tuple)): + raise SBOMLineageError("component references must be a list") + reference_values = list(references or []) + if len(reference_values) > _SL_MAX_REFERENCES: + _sl_truncated(component_truncated, "references") + for item in reference_values[:_SL_MAX_REFERENCES]: + if not isinstance(item, Mapping): + raise SBOMLineageError("component references must contain objects") + safe_references.append(dict(item)) + license_values: list[str] = [] + if licenses is None: + license_values_input: list[Any] = [] + elif isinstance(licenses, (list, tuple, set)): + license_values_input = list(licenses) + else: + license_values_input = [licenses] + if len(license_values_input) > _SL_MAX_LICENSES: + _sl_truncated(component_truncated, "licenses") + if licenses is not None: + for item in license_values_input[:_SL_MAX_LICENSES]: + if isinstance(item, Mapping): + item = item.get("license", item.get("id")) + if isinstance(item, Mapping): + item = item.get("id") or item.get("name") + text = _sl_text(item, "license", limit=128) + if text: + license_values.append(text) + elif licenses: + text = _sl_text(licenses, "license", limit=128) + if text: + license_values.append(text) + unknown = [] + if not normalized_version: + unknown.append("version") + if not _sl_supplier(supplier): + unknown.append("supplier") + if component_truncated: + unknown.append("truncated:" + ",".join(sorted(set(component_truncated)))) + coverage_state = "complete" if normalized_version and _sl_supplier(supplier) and not component_truncated else "partial" + return { + "component_id": normalized_id, + "name": normalized_name, + "version": normalized_version or None, + "supplier": _sl_supplier(supplier) or None, + "identifiers": sorted(ids), + "references": sorted(safe_references, key=lambda item: (item.get("type", ""), item.get("locator", ""))), + "licenses": sorted(set(license_values)), + "component_type": _sl_text(component_type, "component_type", limit=64) or "unknown", + "coverage": {"state": coverage_state, "unknown": sorted(set(unknown)), "truncated": sorted(set(component_truncated))}, + } + + +def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: + source_id = _sl_id(source, "relationship.from") + target_id = _sl_id(target, "relationship.to") + rel_type = _sl_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") + if not isinstance(confidence, str) or confidence not in _SL_CONFIDENCE: + raise SBOMLineageError("relationship confidence is unsupported") + if not isinstance(coverage, str) or coverage not in _SL_COVERAGE: + raise SBOMLineageError("relationship coverage is unsupported") + refs = [] + if evidence_refs is not None and not isinstance(evidence_refs, (list, tuple)): + raise SBOMLineageError("relationship evidence_refs must be a list") + evidence_values = list(evidence_refs or []) + if len(evidence_values) > _SL_MAX_EVIDENCE_REFS: + _sl_truncated(truncated, "evidence_refs") + refs = sorted({_sl_id(ref, "relationship.evidence_ref") for ref in evidence_values[:_SL_MAX_EVIDENCE_REFS]}) + result: dict[str, Any] = { + "from": source_id, + "to": target_id, + "type": rel_type, + "confidence": confidence, + "coverage": coverage, + } + if refs: + result["evidence_refs"] = refs + return result + + +def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: + component_truncated: list[str] = [] + references, identifiers = _sl_spdx_references(raw.get("externalRefs"), truncated=component_truncated) + if truncated is not None: + truncated.extend(item for item in component_truncated if item not in truncated) + return _sl_component( + component_id=raw.get("SPDXID"), + name=raw.get("name"), + version=raw.get("versionInfo"), + supplier=raw.get("supplier"), + identifiers=identifiers, + references=references, + component_type="package", + licenses=raw.get("licenseConcluded"), + truncated=component_truncated, + ) + + +def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: + component_truncated: list[str] = [] + references, identifiers = _sl_cdx_references(raw.get("externalReferences"), truncated=component_truncated) + if "purl" in raw and raw.get("purl") is not None: + identifiers.append(_sl_safe_locator(raw.get("purl"), "purl")) + references.extend(_sl_properties(raw.get("properties"), truncated=component_truncated)) + if truncated is not None: + truncated.extend(item for item in component_truncated if item not in truncated) + return _sl_component( + component_id=raw.get("bom-ref"), + name=raw.get("name"), + version=raw.get("version"), + supplier=raw.get("supplier"), + identifiers=identifiers, + references=references, + component_type=raw.get("type"), + licenses=raw.get("licenses"), + truncated=component_truncated, + ) + + +def _sl_validate_spdx_version(value: Any) -> str: + version_text = _sl_text(value, "SPDX version", required=True, limit=32) + match = _SL_SPDX_VERSION_RE.fullmatch(version_text) + if not match or match.group(1) not in _SL_SPDX_VERSIONS: + raise SBOMLineageError(f"unsupported SPDX version: {version_text}") + return match.group(1) + + +def _sl_validate_cdx_version(value: Any) -> str: + version_text = _sl_text(value, "CycloneDX version", required=True, limit=32) + if version_text not in _SL_CDX_VERSIONS: + raise SBOMLineageError(f"unsupported CycloneDX version: {version_text}") + return version_text + + +def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: + if fmt not in _SL_FORMATS: + raise SBOMLineageError("unsupported SBOM format") + component_ids: set[str] = set() + for item in components: + component_id = item.get("component_id") if isinstance(item, Mapping) else None + if not isinstance(component_id, str) or component_id in component_ids: + raise SBOMLineageError("duplicate or missing component ID") + component_ids.add(component_id) + known_ids = set(component_ids) + if fmt == "SPDX": + known_ids.add(document_id) + dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) + truncation = sorted(set(truncated or [])) + unknown = [] + if not components: + unknown.append("components") + if not relationships: + unknown.append("relationships") + if dangling: + unknown.append("dangling_relationships") + if truncation: + unknown.append("truncated:" + ",".join(truncation)) + if any(item.get("coverage", {}).get("state") != "complete" for item in components): + unknown.append("component_metadata") + if not components: + coverage_state = "unknown" + elif dangling or truncation or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: + coverage_state = "partial" + else: + coverage_state = "complete" + unsigned = { + "schema_version": _SL_SCHEMA, + "format": fmt, + "spec_version": spec_version, + "document_id": document_id, + "document_name": document_name or None, + "document_sha256": hashlib.sha256(raw_bytes).hexdigest(), + "source_ref": source_ref, + "created_at": created_at or None, + "supplier": supplier or None, + "components": sorted(components, key=lambda item: (item["component_id"] == metadata_component_id, item["component_id"])), + "relationships": sorted(relationships, key=lambda item: (item["from"], item["to"], item["type"])), + "coverage": { + "state": coverage_state, + "unknown": sorted(set(unknown)), + "component_count": len(components), + "relationship_count": len(relationships), + "truncated": truncation, + "dangling_relationships": dangling, + }, + } + unsigned["ingestion_digest"] = _sl_sha(unsigned) + return unsigned + + +def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + version = _sl_validate_spdx_version(value.get("spdxVersion")) + document_id = _sl_id(value.get("SPDXID"), "SPDXID") + creation = value.get("creationInfo", {}) + if creation is None: + creation = {} + if not isinstance(creation, Mapping): + raise SBOMLineageError("creationInfo must be an object") + creators = creation.get("creators", []) + if not isinstance(creators, list): + raise SBOMLineageError("creationInfo.creators must be a list") + supplier = _sl_supplier(creators[0] if creators else "") + packages = _sl_list(value.get("packages"), "packages") + raw_relationships = _sl_list(value.get("relationships"), "relationships") + truncated = [] + if len(packages) > _SL_MAX_COMPONENTS: + truncated.append("packages") + if len(raw_relationships) > _SL_MAX_RELATIONSHIPS: + truncated.append("relationships") + components = [] + for item in packages[:_SL_MAX_COMPONENTS]: + if not isinstance(item, Mapping): + raise SBOMLineageError("packages must contain objects") + components.append(_sl_spdx_component(item, truncated=truncated)) + relationships = [] + for raw in raw_relationships[:_SL_MAX_RELATIONSHIPS]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("relationships must contain objects") + relationships.append(_sl_relationship( + raw.get("spdxElementId"), raw.get("relatedSpdxElement"), + raw.get("relationshipType", "related"), truncated=truncated, + )) + return _sl_finalize_document( + fmt="SPDX", spec_version=version, document_id=document_id, + document_name=_sl_text(value.get("name"), "document_name"), + created_at=_sl_text(creation.get("created"), "created_at"), supplier=supplier, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + ) + + +def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + if value.get("bomFormat") != "CycloneDX": + raise SBOMLineageError("unsupported SBOM format") + version = _sl_validate_cdx_version(value.get("specVersion")) + metadata = value.get("metadata", {}) + if metadata is None: + metadata = {} + if not isinstance(metadata, Mapping): + raise SBOMLineageError("metadata must be an object") + metadata_component = metadata.get("component") + if metadata_component is not None and not isinstance(metadata_component, Mapping): + raise SBOMLineageError("metadata.component must be an object") + raw_components = [] + truncated = [] + if metadata_component: + raw_components.append(metadata_component) + raw_component_list = _sl_list(value.get("components"), "components") + raw_dependency_list = _sl_list(value.get("dependencies"), "dependencies") + if len(raw_component_list) > _SL_MAX_COMPONENTS: + truncated.append("components") + if len(raw_dependency_list) > _SL_MAX_RELATIONSHIPS: + truncated.append("dependencies") + for item in raw_component_list[:_SL_MAX_COMPONENTS]: + if not isinstance(item, Mapping): + raise SBOMLineageError("components must contain objects") + raw_components.append(item) + components: list[dict[str, Any]] = [] + for raw in raw_components: + components.append(_sl_cdx_component(raw, truncated=truncated)) + relationships = [] + for raw in raw_dependency_list[:_SL_MAX_RELATIONSHIPS]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("dependencies must contain objects") + source = raw.get("ref") + targets = _sl_list(raw.get("dependsOn"), "dependsOn") + if len(targets) > _SL_MAX_DEPENDENCY_TARGETS: + truncated.append("dependency_edges") + for target in targets[:_SL_MAX_DEPENDENCY_TARGETS]: + relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated)) + creators = metadata.get("authors", []) + if not isinstance(creators, list): + raise SBOMLineageError("metadata.authors must be a list") + supplier = _sl_supplier(creators[0] if creators else (metadata_component or {})) + document_id = _sl_id(value.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + return _sl_finalize_document( + fmt="CycloneDX", spec_version=version, document_id=document_id, + document_name="CycloneDX BOM", created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + metadata_component_id=components[0]["component_id"] if metadata_component else "", + ) + + +def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + documents = _sl_descendants(root, "SpdxDocument") + document = documents[0] if documents else root + version = _sl_validate_spdx_version(_sl_xml_value(document, "spdxVersion")) + raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") + if not raw_document_id: + raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") + document_id = _sl_id(raw_document_id, "SPDXID") + creation = next(iter(_sl_descendants(document, "creationInfo")), None) + created_at = _sl_xml_value(creation, "created") + creator = _sl_xml_value(creation, "creator") + packages = _sl_descendants(document, "Package", "package") + relationships_raw = _sl_descendants(document, "Relationship", "relationship") + truncated = [] + if len(packages) > _SL_MAX_COMPONENTS: + truncated.append("packages") + if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: + truncated.append("relationships") + components = [] + for raw in packages[:_SL_MAX_COMPONENTS]: + component_truncated: list[str] = [] + refs = [] + identifiers = [] + for ref in _sl_descendants(raw, "ExternalRef", "externalRef"): + locator = _sl_xml_value(ref, "referenceLocator") + if not locator: + raise SBOMLineageError("SPDX externalRef requires referenceLocator") + ref_type = _sl_xml_value(ref, "referenceType", default="other") + item = _sl_normalize_reference( + ref_type, locator, category=_sl_xml_value(ref, "referenceCategory"), + truncated=component_truncated, + ) + refs.append(item) + if ref_type.casefold() == "purl": + identifiers.append(item["locator"]) + truncated.extend(item for item in component_truncated if item not in truncated) + component_id = _sl_xml_value(raw, "SPDXID", "spdxid") + if not component_id: + component_id = next((str(value).lstrip("#") for key, value in raw.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") + components.append(_sl_component( + component_id=component_id, + name=_sl_xml_value(raw, "name"), + version=_sl_xml_value(raw, "versionInfo"), + supplier=_sl_xml_value(raw, "supplier"), + identifiers=identifiers, + references=refs, + component_type="package", + licenses=_sl_xml_value(raw, "licenseConcluded"), + truncated=component_truncated, + )) + relationships = [] + for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: + relationships.append(_sl_relationship( + _sl_xml_value(raw, "spdxElementId"), + _sl_xml_value(raw, "relatedSpdxElement"), + _sl_xml_value(raw, "relationshipType", default="related"), + truncated=truncated, + )) + return _sl_finalize_document( + fmt="SPDX", spec_version=version, document_id=document_id, + document_name=_sl_xml_value(document, "name"), created_at=created_at, supplier=creator, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + ) + + +def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) + document_id = _sl_id(_sl_xml_text(root, "SPDXID"), "SPDXID") + creation = _sl_child(root, "creationInfo") + created_at = _sl_xml_text(creation, "created") if creation is not None else "" + creator = _sl_xml_text(creation, "creator") if creation is not None else "" + packages = _sl_children(root, "package") + relationships_raw = _sl_children(root, "relationship") + truncated: list[str] = [] + if len(packages) > _SL_MAX_COMPONENTS: + truncated.append("packages") + if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: + truncated.append("relationships") + components = [] + for raw in packages[:_SL_MAX_COMPONENTS]: + component_truncated: list[str] = [] + refs = [] + identifiers = [] + for ref in _sl_children(raw, "externalRef"): + locator = _sl_xml_text(ref, "referenceLocator") + ref_type = _sl_xml_text(ref, "referenceType", default="other") + if not locator: + raise SBOMLineageError("SPDX externalRef requires referenceLocator") + item = _sl_normalize_reference( + ref_type, locator, category=_sl_xml_text(ref, "referenceCategory"), + truncated=component_truncated, + ) + refs.append(item) + if ref_type.casefold() == "purl": + identifiers.append(item["locator"]) + truncated.extend(item for item in component_truncated if item not in truncated) + components.append(_sl_component( + component_id=_sl_xml_text(raw, "SPDXID"), name=_sl_xml_text(raw, "name"), + version=_sl_xml_text(raw, "versionInfo"), supplier=_sl_xml_text(raw, "supplier"), + identifiers=identifiers, references=refs, component_type="package", + licenses=_sl_xml_text(raw, "licenseConcluded"), + truncated=component_truncated, + )) + relationships = [] + for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: + relationships.append(_sl_relationship( + _sl_xml_text(raw, "spdxElementId"), _sl_xml_text(raw, "relatedSpdxElement"), + _sl_xml_text(raw, "relationshipType", default="related"), + truncated=truncated, + )) + return _sl_finalize_document( + fmt="SPDX", spec_version=version, document_id=document_id, + document_name=_sl_xml_text(root, "name"), created_at=created_at, supplier=creator, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + ) + + +def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None) -> dict[str, Any]: + component_truncated: list[str] = [] + refs = [] + identifiers = [] + purl = _sl_xml_text(raw, "purl") + if purl: + identifiers.append(_sl_safe_locator(purl, "purl")) + external = _sl_child(raw, "externalReferences") + if external is not None: + reference_nodes = _sl_children(external, "reference") + if len(reference_nodes) > _SL_MAX_REFERENCES: + component_truncated.append("externalReferences") + for ref in reference_nodes[:_SL_MAX_REFERENCES]: + locator = _sl_xml_text(ref, "url") + if not locator: + raise SBOMLineageError("CycloneDX externalReference requires url") + refs.append(_sl_normalize_reference( + _sl_xml_text(ref, "type", default="other"), locator, + truncated=component_truncated, + )) + licenses = [] + license_container = _sl_child(raw, "licenses") + if license_container is not None: + license_nodes = _sl_children(license_container, "license") + if len(license_nodes) > _SL_MAX_LICENSES: + component_truncated.append("licenses") + for item in license_nodes[:_SL_MAX_LICENSES]: + licenses.append(_sl_xml_text(item, "id") or _sl_xml_text(item, "name")) + if truncated is not None: + truncated.extend(item for item in component_truncated if item not in truncated) + return _sl_component( + component_id=raw.attrib.get("bom-ref"), name=_sl_xml_text(raw, "name"), + version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_child(raw, "supplier"), "name"), + identifiers=identifiers, references=refs, component_type=raw.attrib.get("type"), licenses=licenses, + truncated=component_truncated, + ) + + +def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + namespace = root.tag.split("}", 1)[0].lstrip("{") if "}" in root.tag else "" + match = _SL_CDX_NAMESPACE_RE.search(namespace) + if not match: + raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") + version = _sl_validate_cdx_version(match.group(1)) + metadata = _sl_child(root, "metadata") + metadata_component = _sl_child(metadata, "component") if metadata is not None else None + raw_components = [] + truncated: list[str] = [] + if metadata_component is not None: + raw_components.append(metadata_component) + components_node = _sl_child(root, "components") + if components_node is not None: + component_nodes = _sl_children(components_node, "component") + if len(component_nodes) > _SL_MAX_COMPONENTS: + truncated.append("components") + raw_components.extend(component_nodes[:_SL_MAX_COMPONENTS]) + components = [] + for raw in raw_components: + components.append(_sl_cdx_xml_component(raw, truncated=truncated)) + relationships = [] + dependencies_node = _sl_child(root, "dependencies") + if dependencies_node is not None: + dependency_nodes = _sl_children(dependencies_node, "dependency") + if len(dependency_nodes) > _SL_MAX_RELATIONSHIPS: + truncated.append("dependencies") + for dependency in dependency_nodes[:_SL_MAX_RELATIONSHIPS]: + source = dependency.attrib.get("ref") + children = _sl_children(dependency, "dependency") + if len(children) > _SL_MAX_DEPENDENCY_TARGETS: + truncated.append("dependency_edges") + for child in children[:_SL_MAX_DEPENDENCY_TARGETS]: + relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated)) + timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" + authors = _sl_child(metadata, "authors") if metadata is not None else None + author = _sl_xml_text(_sl_child(authors, "author"), "name") if authors is not None else "" + document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + return _sl_finalize_document( + fmt="CycloneDX", spec_version=version, document_id=document_id, + document_name="CycloneDX BOM", created_at=timestamp, supplier=author, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + metadata_component_id=components[0]["component_id"] if metadata_component is not None else "", + ) + + +def _sl_read_bounded(path: Path) -> bytes: + """Read a file only after checking its size, including race re-check.""" + try: + size = path.stat().st_size + except OSError as exc: + raise SBOMLineageError(f"could not stat SBOM: {exc}") from exc + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + try: + raw_bytes = path.read_bytes() + except OSError as exc: + raise SBOMLineageError(f"could not read SBOM: {exc}") from exc + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return raw_bytes + + +def _sl_payload(document: Any) -> tuple[Any, bytes]: + if isinstance(document, Path): + return _sl_payload(_sl_read_bounded(document)) + if isinstance(document, bytes): + raw_bytes = document + elif isinstance(document, bytearray): + raw_bytes = bytes(document) + elif isinstance(document, Mapping): + try: + raw_bytes = _sl_json(document).encode("utf-8") + except (TypeError, ValueError) as exc: + raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc + elif isinstance(document, str): + possible_path = Path(document) + if "\n" not in document and possible_path.exists(): + return _sl_payload(possible_path) + raw_bytes = document.encode("utf-8") + else: + raise SBOMLineageError("SBOM input must be bytes, text, path, or object") + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + if not raw_bytes.strip(): + raise SBOMLineageError("SBOM input is empty") + try: + value = json.loads(raw_bytes.decode("utf-8")) + if not isinstance(value, Mapping): + raise SBOMLineageError("SBOM JSON root must be an object") + return dict(value), raw_bytes + except (UnicodeDecodeError, json.JSONDecodeError): + try: + root = _sl_et.fromstring(raw_bytes) + _sl_validate_xml_tree(root) + except _sl_et.ParseError as exc: + raise SBOMLineageError("SBOM is neither valid JSON nor XML") from exc + return root, raw_bytes + + +def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, Any]: + """Parse one SPDX or CycloneDX JSON/XML document into a safe projection.""" + value, raw_bytes = _sl_payload(document) + normalized_source = _sl_safe_source_ref(source_ref, raw_bytes) + if isinstance(value, Mapping): + if value.get("spdxVersion") is not None: + return _sl_parse_spdx_json(value, raw_bytes, normalized_source) + if value.get("bomFormat") is not None: + return _sl_parse_cdx_json(value, raw_bytes, normalized_source) + raise SBOMLineageError("SBOM format is missing or unsupported") + root_name = _sl_local(value).casefold() + if root_name == "spdxdocument": + if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): + return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) + return _sl_spdx_xml(value, raw_bytes, normalized_source) + if root_name == "rdf" and _sl_descendants(value, "SpdxDocument"): + return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) + if root_name == "bom": + return _sl_parse_cdx_xml(value, raw_bytes, normalized_source) + raise SBOMLineageError("SBOM XML format is missing or unsupported") + + +def _sl_node_kind(node_id: str) -> str: + if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("document:"): + return "document" + if node_id.startswith("artifact:"): + return "artifact" + if node_id.startswith("deployment:"): + return "deployment" + if node_id.startswith("build:"): + return "build" + if node_id.startswith("source:"): + return "source" + return "component" + + +def _sl_require_keys(value: Any, required: set[str], optional: set[str], field: str) -> None: + if not isinstance(value, Mapping) or not required.issubset(set(value)) or set(value) - required - optional: + raise SBOMLineageError(f"{field} schema is invalid") + + +def _sl_string_list(value: Any, field: str, *, maximum: int) -> list[str]: + items = _sl_list(value, field) + if len(items) > maximum: + raise SBOMLineageError(f"{field} exceeds {maximum} items") + result = [] + for item in items: + text = _sl_strict_text(item, field, limit=1024) + assert text is not None + result.append(text) + if result != sorted(set(result)): + raise SBOMLineageError(f"{field} must be unique and sorted") + return result + + +def _sl_validate_reference(reference: Any) -> dict[str, Any]: + _sl_require_keys(reference, {"type", "locator"}, {"category", "hashes"}, "reference") + ref_type = _sl_strict_text(reference.get("type"), "reference.type", limit=64) + assert ref_type is not None + if ref_type not in _SL_REFERENCE_TYPES: + raise SBOMLineageError("reference.type is unsupported") + locator = _sl_strict_safe_locator(reference.get("locator"), "reference.locator") + result: dict[str, Any] = {"type": ref_type, "locator": locator} + if "category" in reference: + category = _sl_strict_text(reference.get("category"), "reference.category", limit=64) + assert category is not None + result["category"] = category + if "hashes" in reference: + hashes = _sl_list(reference.get("hashes"), "reference.hashes") + if not hashes or len(hashes) > _SL_MAX_HASHES: + raise SBOMLineageError("reference.hashes is outside its bounds") + checked_hashes = [] + for item in hashes: + _sl_require_keys(item, {"alg", "content"}, set(), "reference.hash") + algorithm = _sl_strict_text(item.get("alg"), "reference.hash.alg", limit=32) + content = _sl_strict_text(item.get("content"), "reference.hash.content", limit=256) + assert algorithm is not None and content is not None + if not re.fullmatch(r"[0-9a-f]{32,128}", content): + raise SBOMLineageError("reference.hash.content is not a digest") + checked_hashes.append({"alg": algorithm, "content": content}) + result["hashes"] = checked_hashes + return result + + +def _sl_validate_component_coverage(value: Any, *, has_version: bool, has_supplier: bool) -> dict[str, Any]: + _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "component.coverage") + state = value.get("state") + if not isinstance(state, str) or state not in _SL_COVERAGE: + raise SBOMLineageError("component.coverage.state is invalid") + unknown = _sl_string_list(value.get("unknown"), "component.coverage.unknown", maximum=32) + truncated = _sl_string_list(value.get("truncated"), "component.coverage.truncated", maximum=32) + expected_unknown = [] + if not has_version: + expected_unknown.append("version") + if not has_supplier: + expected_unknown.append("supplier") + if truncated: + expected_unknown.append("truncated:" + ",".join(truncated)) + expected_state = "complete" if not expected_unknown else "partial" + if state != expected_state or unknown != sorted(set(expected_unknown)): + raise SBOMLineageError("component coverage is inconsistent with its fields") + return {"state": state, "unknown": unknown, "truncated": truncated} + + +def _sl_validate_component(component: Any) -> dict[str, Any]: + required = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} + _sl_require_keys(component, required, set(), "component") + component_id_text = _sl_strict_text(component.get("component_id"), "component.component_id", limit=256) + assert component_id_text is not None + component_id = _sl_id(component_id_text, "component.component_id") + name = _sl_strict_text(component.get("name"), "component.name", limit=256) + version = _sl_strict_text(component.get("version"), "component.version", allow_none=True, limit=128) + supplier = _sl_strict_text(component.get("supplier"), "component.supplier", allow_none=True, limit=512) + identifiers = _sl_string_list(component.get("identifiers"), "component.identifiers", maximum=_SL_MAX_IDENTIFIERS) + if component_id not in identifiers: + raise SBOMLineageError("component.identifiers must bind component_id") + for identifier in identifiers: + _sl_strict_safe_locator(identifier, "component.identifier") + references = _sl_list(component.get("references"), "component.references") + if len(references) > _SL_MAX_REFERENCES: + raise SBOMLineageError("component.references exceeds its bound") + checked_references = [_sl_validate_reference(reference) for reference in references] + licenses = _sl_string_list(component.get("licenses"), "component.licenses", maximum=_SL_MAX_LICENSES) + component_type = _sl_strict_text(component.get("component_type"), "component.component_type", limit=64) + assert component_type is not None + coverage = _sl_validate_component_coverage( + component.get("coverage"), has_version=version is not None, has_supplier=supplier is not None, + ) + checked = { + "component_id": component_id, + "name": name, + "version": version, + "supplier": supplier, + "identifiers": identifiers, + "references": checked_references, + "licenses": licenses, + "component_type": component_type, + "coverage": coverage, + } + if _sl_json(checked) != _sl_json(dict(component)): + raise SBOMLineageError("component projection is not normalized") + return checked + + +def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") -> dict[str, Any]: + _sl_require_keys(relationship, {"from", "to", "type", "confidence", "coverage"}, {"evidence_refs"}, field) + source = _sl_strict_text(relationship.get("from"), f"{field}.from", limit=256) + target = _sl_strict_text(relationship.get("to"), f"{field}.to", limit=256) + rel_type = _sl_strict_text(relationship.get("type"), f"{field}.type", limit=96) + confidence = relationship.get("confidence") + coverage = relationship.get("coverage") + assert source is not None and target is not None and rel_type is not None + _sl_id(source, f"{field}.from") + _sl_id(target, f"{field}.to") + if rel_type != rel_type.casefold().replace(" ", "_"): + raise SBOMLineageError(f"{field}.type is not normalized") + if confidence not in _SL_CONFIDENCE or coverage not in _SL_COVERAGE: + raise SBOMLineageError(f"{field} confidence or coverage is invalid") + result: dict[str, Any] = { + "from": source, "to": target, "type": rel_type, + "confidence": confidence, "coverage": coverage, + } + if "evidence_refs" in relationship: + evidence_refs = _sl_string_list(relationship.get("evidence_refs"), f"{field}.evidence_refs", maximum=_SL_MAX_EVIDENCE_REFS) + if not evidence_refs: + raise SBOMLineageError(f"{field}.evidence_refs cannot be empty") + for ref in evidence_refs: + _sl_id(ref, f"{field}.evidence_ref") + result["evidence_refs"] = evidence_refs + if _sl_json(result) != _sl_json(dict(relationship)): + raise SBOMLineageError(f"{field} is not normalized") + return result + + +def _sl_expected_document_coverage(document_id: str, fmt: str, components: list[Mapping[str, Any]], relationships: list[Mapping[str, Any]], truncated: list[str]) -> dict[str, Any]: + component_ids = {item["component_id"] for item in components} + known_ids = set(component_ids) + if fmt == "SPDX": + known_ids.add(document_id) + dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) + unknown: list[str] = [] + if not components: + unknown.append("components") + if not relationships: + unknown.append("relationships") + if dangling: + unknown.append("dangling_relationships") + if truncated: + unknown.append("truncated:" + ",".join(sorted(set(truncated)))) + if any(item["coverage"]["state"] != "complete" for item in components): + unknown.append("component_metadata") + state = "unknown" if not components else "partial" if (dangling or truncated or not relationships or any(item["coverage"]["state"] != "complete" for item in components)) else "complete" + return { + "state": state, + "unknown": sorted(set(unknown)), + "component_count": len(components), + "relationship_count": len(relationships), + "truncated": sorted(set(truncated)), + "dangling_relationships": dangling, + } + + +def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: + required = { + "schema_version", "format", "spec_version", "document_id", "document_name", + "document_sha256", "source_ref", "created_at", "supplier", "components", + "relationships", "coverage", "ingestion_digest", + } + if not isinstance(document, Mapping) or set(document) != required or document.get("schema_version") != _SL_SCHEMA: + raise SBOMLineageError("normalized SBOM document shape is invalid") + supplied = document.get("ingestion_digest") + unsigned = dict(document) + unsigned.pop("ingestion_digest", None) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + raise SBOMLineageError("normalized SBOM ingestion digest mismatch") + fmt = document.get("format") + if fmt not in _SL_FORMATS: + raise SBOMLineageError("normalized SBOM format is invalid") + spec_version = document.get("spec_version") + if fmt == "SPDX": + _sl_validate_spdx_version(f"SPDX-{spec_version}") + else: + _sl_validate_cdx_version(spec_version) + document_id_text = _sl_strict_text(document.get("document_id"), "document_id", limit=256) + assert document_id_text is not None + document_id = _sl_id(document_id_text, "document_id") + for field, limit in (("document_name", 256), ("created_at", 128), ("supplier", 512)): + _sl_strict_text(document.get(field), field, allow_none=True, limit=limit) + document_sha = document.get("document_sha256") + if not isinstance(document_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", document_sha): + raise SBOMLineageError("normalized SBOM document digest is invalid") + _sl_strict_source_ref(document.get("source_ref")) + components = _sl_list(document.get("components"), "components") + relationships = _sl_list(document.get("relationships"), "relationships") + if len(components) > _SL_MAX_COMPONENTS or len(relationships) > _SL_MAX_RELATIONSHIPS: + raise SBOMLineageError("normalized SBOM collection exceeds its bound") + checked_components = [] + seen: set[str] = set() + for component in components: + checked = _sl_validate_component(component) + if checked["component_id"] in seen: + raise SBOMLineageError("normalized SBOM contains duplicate component IDs") + seen.add(checked["component_id"]) + checked_components.append(checked) + checked_relationships = [_sl_validate_relationship(item) for item in relationships] + coverage = document.get("coverage") + _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") + if coverage.get("state") not in _SL_COVERAGE: + raise SBOMLineageError("document.coverage.state is invalid") + _sl_string_list(coverage.get("unknown"), "document.coverage.unknown", maximum=64) + truncated = _sl_string_list(coverage.get("truncated"), "document.coverage.truncated", maximum=64) + dangling = _sl_string_list(coverage.get("dangling_relationships"), "document.coverage.dangling_relationships", maximum=_SL_MAX_RELATIONSHIPS) + _sl_nonnegative_int(coverage.get("component_count"), "document.coverage.component_count", maximum=_SL_MAX_COMPONENTS) + _sl_nonnegative_int(coverage.get("relationship_count"), "document.coverage.relationship_count", maximum=_SL_MAX_RELATIONSHIPS) + expected = _sl_expected_document_coverage(document_id, fmt, checked_components, checked_relationships, truncated) + if coverage != expected: + raise SBOMLineageError("document coverage is inconsistent with its contents") + return dict(document) + + +def verify_sbom_document(document: Mapping[str, Any]) -> dict[str, Any]: + try: + checked = _sl_validate_document(document) + except (SBOMLineageError, TypeError, ValueError) as exc: + return {"valid": False, "error": str(exc)} + return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} + + +def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: + if edges is None: + return [] + if not isinstance(edges, (list, tuple)): + raise SBOMLineageError("lineage edges must be a list") + result = [] + if len(edges) > _SL_MAX_EDGES: + _sl_truncated(truncated, "edges") + for raw in edges[:_SL_MAX_EDGES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("lineage edges must contain objects") + result.append(_sl_relationship( + raw.get("from", raw.get("source")), raw.get("to", raw.get("target")), raw.get("type", "related"), + confidence=raw.get("confidence", "unknown"), coverage=raw.get("coverage", "unknown"), + evidence_refs=raw.get("evidence_refs", []), truncated=truncated, + )) + return sorted(result, key=lambda item: (item["from"], item["to"], item["type"], item["confidence"], item["coverage"])) + + +def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: + """Build a deterministic graph from normalized or raw SBOM documents.""" + if not isinstance(documents, (list, tuple)) or not documents: + raise SBOMLineageError("at least one SBOM document is required") + truncated: list[str] = [] + if len(documents) > _SL_MAX_DOCUMENTS: + truncated.append("documents") + normalized = [] + for document in documents[:_SL_MAX_DOCUMENTS]: + if isinstance(document, Mapping) and document.get("schema_version") == _SL_SCHEMA: + normalized.append(_sl_validate_document(document)) + else: + normalized.append(ingest_sbom_document(document)) + nodes: dict[str, dict[str, Any]] = {} + component_fingerprints: dict[str, str] = {} + native_edges: list[dict[str, Any]] = [] + for document in normalized: + for component in document.get("components", []): + component_id = component["component_id"] + fingerprint = _sl_sha(component) + if component_id in component_fingerprints and component_fingerprints[component_id] != fingerprint: + raise SBOMLineageError(f"conflicting duplicate component ID: {component_id}") + if component_id in component_fingerprints: + continue + component_fingerprints[component_id] = fingerprint + node = dict(component) + node["node_id"] = component_id + node["kind"] = "component" + node["document_sha256"] = document["document_sha256"] + nodes[component_id] = node + native_edges.extend(document.get("relationships", [])) + for marker in document.get("coverage", {}).get("truncated", []): + _sl_truncated(truncated, marker) + external_edges = [] if edges is None else list(edges) if isinstance(edges, (list, tuple)) else None + if external_edges is None: + raise SBOMLineageError("lineage edges must be a list") + all_edges = _sl_lineage_edges(native_edges + external_edges, truncated=truncated) + for edge in all_edges: + for node_id in (edge["from"], edge["to"]): + nodes.setdefault(node_id, {"node_id": node_id, "kind": _sl_node_kind(node_id), "coverage": {"state": "partial", "unknown": ["node_metadata"], "truncated": []}}) + coverage_states = {edge["coverage"] for edge in all_edges} + document_states = {document.get("coverage", {}).get("state", "unknown") for document in normalized} + node_states = {node.get("coverage", {}).get("state", "complete") for node in nodes.values() if node.get("kind") == "component"} + if "unknown" in coverage_states or "unknown" in document_states or "unknown" in node_states: + coverage_state = "unknown" + elif truncated or "partial" in coverage_states or "partial" in document_states or "partial" in node_states: + coverage_state = "partial" + else: + coverage_state = "complete" + body: dict[str, Any] = { + "schema_version": _SL_LINEAGE_SCHEMA, + "documents": sorted(normalized, key=lambda item: item["document_sha256"]), + "nodes": [nodes[key] for key in sorted(nodes)], + "edges": all_edges, + "coverage": {"state": coverage_state, "unknown": [] if coverage_state == "complete" else ["external_lineage"], "truncated": sorted(set(truncated))}, + } + body["lineage_digest"] = _sl_sha(body) + return body + + +def _sl_validate_node_coverage(value: Any, *, component: bool) -> dict[str, Any]: + _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "node.coverage") + state = value.get("state") + if state not in _SL_COVERAGE: + raise SBOMLineageError("node.coverage.state is invalid") + unknown = _sl_string_list(value.get("unknown"), "node.coverage.unknown", maximum=32) + truncated = _sl_string_list(value.get("truncated"), "node.coverage.truncated", maximum=32) + if not component and (state != "partial" or unknown != ["node_metadata"] or truncated): + raise SBOMLineageError("external lineage node coverage is invalid") + return {"state": state, "unknown": unknown, "truncated": truncated} + + +def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[str, Mapping[str, Any]]]]) -> str: + if not isinstance(node, Mapping): + raise SBOMLineageError("lineage node must be an object") + node_id = _sl_strict_text(node.get("node_id"), "node.node_id", limit=256) + kind = _sl_strict_text(node.get("kind"), "node.kind", limit=32) + assert node_id is not None and kind is not None + _sl_id(node_id, "node.node_id") + expected_kind = _sl_node_kind(node_id) + if kind != expected_kind: + raise SBOMLineageError("lineage node kind does not match its ID") + component_fields = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} + if kind == "component" and component_fields.issubset(set(node)): + required = component_fields | {"node_id", "kind", "document_sha256"} + _sl_require_keys(node, required, set(), "component node") + component = {key: node[key] for key in component_fields} + checked = _sl_validate_component(component) + if checked["component_id"] != node_id: + raise SBOMLineageError("component node_id is not bound to component_id") + document_sha = node.get("document_sha256") + if not isinstance(document_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", document_sha): + raise SBOMLineageError("component node document digest is invalid") + candidates = component_map.get(node_id, []) + if not any(document_sha == digest and _sl_json(checked) == _sl_json(dict(source)) for digest, source in candidates): + raise SBOMLineageError("component node is not bound to a source document") + return node_id + if kind == "component": + _sl_require_keys(node, {"node_id", "kind", "coverage"}, set(), "component stub") + _sl_validate_node_coverage(node.get("coverage"), component=False) + if node_id in component_map: + raise SBOMLineageError("document-backed component node is incomplete") + return node_id + _sl_require_keys(node, {"node_id", "kind", "coverage"}, set(), "external node") + _sl_validate_node_coverage(node.get("coverage"), component=False) + return node_id + + +def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: + required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} + if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: + raise SBOMLineageError("unsupported software-lineage schema") + supplied = lineage.get("lineage_digest") + unsigned = dict(lineage) + unsigned.pop("lineage_digest", None) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + raise SBOMLineageError("lineage digest mismatch") + documents = _sl_list(lineage.get("documents"), "lineage.documents") + nodes = _sl_list(lineage.get("nodes"), "lineage.nodes") + edges = _sl_list(lineage.get("edges"), "lineage.edges") + if len(documents) > _SL_MAX_DOCUMENTS or len(edges) > _SL_MAX_EDGES or len(nodes) > _SL_MAX_NODES: + raise SBOMLineageError("lineage collection exceeds its bound") + checked_documents = [_sl_validate_document(document) for document in documents] + component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} + for document in checked_documents: + for component in document["components"]: + component_map.setdefault(component["component_id"], []).append((document["document_sha256"], component)) + for component_id, candidates in component_map.items(): + if len({_sl_sha(item) for _, item in candidates}) > 1: + raise SBOMLineageError(f"conflicting component projection for {component_id}") + node_ids: set[str] = set() + full_component_ids: set[str] = set() + for node in nodes: + node_id = _sl_validate_lineage_node(node, component_map) + if node_id in node_ids: + raise SBOMLineageError("lineage contains duplicate nodes") + node_ids.add(node_id) + if isinstance(node, Mapping) and "component_id" in node: + full_component_ids.add(node_id) + if full_component_ids != set(component_map): + raise SBOMLineageError("lineage component nodes are not bound to all documents") + checked_edges = [] + for edge in edges: + checked = _sl_validate_relationship(edge, field="lineage.edge") + if checked["from"] not in node_ids or checked["to"] not in node_ids: + raise SBOMLineageError("lineage edge endpoint is not bound to a node") + checked_edges.append(checked) + coverage = lineage.get("coverage") + _sl_require_keys(coverage, {"state", "unknown", "truncated"}, set(), "lineage.coverage") + state = coverage.get("state") + if state not in _SL_COVERAGE: + raise SBOMLineageError("lineage.coverage.state is invalid") + unknown = _sl_string_list(coverage.get("unknown"), "lineage.coverage.unknown", maximum=32) + truncated = _sl_string_list(coverage.get("truncated"), "lineage.coverage.truncated", maximum=64) + nested_truncated = {item for document in checked_documents for item in document["coverage"]["truncated"]} + if not nested_truncated.issubset(set(truncated)): + raise SBOMLineageError("lineage coverage lost document truncation metadata") + edge_states = {edge["coverage"] for edge in checked_edges} + document_states = {document["coverage"]["state"] for document in checked_documents} + node_states = {node["coverage"]["state"] for node in nodes if node.get("kind") == "component"} + expected_state = "unknown" if "unknown" in edge_states or "unknown" in document_states or "unknown" in node_states else "partial" if truncated or "partial" in edge_states or "partial" in document_states or "partial" in node_states else "complete" + expected_unknown = [] if expected_state == "complete" else ["external_lineage"] + if state != expected_state or unknown != expected_unknown: + raise SBOMLineageError("lineage coverage is inconsistent with its contents") + return dict(lineage) + + +def verify_sbom_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: + try: + loaded = _sl_loaded_lineage(lineage) + except (SBOMLineageError, TypeError, ValueError) as exc: + return {"valid": False, "error": str(exc)} + return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} + + +def _sl_query_matches(nodes: list[Mapping[str, Any]], query: str) -> list[str]: + term = _sl_strict_text(query, "query", limit=512) + assert term is not None + term = term.casefold() + matches = [] + for node in nodes: + values = [node.get("node_id", ""), node.get("name", ""), node.get("version", "") or ""] + values.extend(node.get("identifiers", []) if isinstance(node.get("identifiers"), list) else []) + references = node.get("references", []) + values.extend(ref.get("locator", "") for ref in references if isinstance(ref, Mapping)) + if any(term in str(value).casefold() for value in values): + matches.append(str(node["node_id"])) + return sorted(set(matches)) + + +def _sl_path_state(path: list[Mapping[str, Any]]) -> str: + states = {str(edge.get("coverage", "unknown")) for edge in path} + if "unknown" in states: + return "unknown" + if "partial" in states: + return "partial" + return "complete" + + +def _sl_path_confidence(path: list[Mapping[str, Any]]) -> str: + states = {str(edge.get("confidence", "unknown")) for edge in path} + if "unknown" in states: + return "unknown" + if "low" in states: + return "low" + if "medium" in states: + return "medium" + return "high" + + +def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32) -> dict[str, Any]: + """Find impacted artifact nodes and return every traversed evidence edge.""" + loaded = _sl_loaded_lineage(lineage) + limit = _sl_limit(limit, "limit") + query_text = _sl_strict_text(query, "query", limit=512) + assert query_text is not None + nodes = loaded.get("nodes", []) + edges = loaded.get("edges", []) + if not isinstance(nodes, list) or not isinstance(edges, list): + raise SBOMLineageError("lineage nodes and edges must be lists") + matches = _sl_query_matches(nodes, query_text) + truncated: list[str] = [] + if len(matches) > _SL_MAX_QUERY_MATCHES: + truncated.append("matched_nodes") + matches = matches[:_SL_MAX_QUERY_MATCHES] + adjacency: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} + for edge in edges: + if not isinstance(edge, Mapping): + continue + source, target = str(edge.get("from")), str(edge.get("to")) + adjacency.setdefault(source, []).append((target, edge)) + adjacency.setdefault(target, []).append((source, edge)) + found: dict[str, dict[str, Any]] = {} + path_truncated = False + for start in matches: + queue = deque([(start, [], {start})]) + while queue: + current, path, visited = queue.popleft() + if current != start and _sl_node_kind(current) == "artifact": + candidate = {"artifact_id": current, "path": path, "coverage": _sl_path_state(path), "confidence": _sl_path_confidence(path), "evidence_refs": sorted({ref for edge in path for ref in edge.get("evidence_refs", [])})} + previous = found.get(current) + if previous is None or len(candidate["path"]) < len(previous["path"]): + found[current] = candidate + continue + if len(path) >= _SL_MAX_PATH_LENGTH: + if adjacency.get(current): + path_truncated = True + continue + for neighbor, edge in sorted(adjacency.get(current, []), key=lambda item: (item[0], item[1].get("type", ""))): + if neighbor in visited: + continue + queue.append((neighbor, path + [dict(edge)], visited | {neighbor})) + all_impacted = [found[key] for key in sorted(found)] + if len(all_impacted) > limit: + truncated.append("impacted_artifacts") + impacted = all_impacted[:limit] + if path_truncated: + truncated.append("path") + global_state = loaded.get("coverage", {}).get("state", "unknown") + if truncated and global_state == "complete": + global_state = "partial" + path_states = {item["coverage"] for item in impacted} + if impacted: + status = "unknown" if "unknown" in path_states or global_state == "unknown" else "partial" if "partial" in path_states or global_state == "partial" else "complete" + if truncated or "unknown" in path_states: + impact_status = "not_established" + elif "partial" in path_states: + impact_status = "partial" + else: + impact_status = "established" + else: + status = "unknown" if global_state != "complete" else "not_found" + impact_status = "not_established" + unsigned: dict[str, Any] = { + "schema_version": _SL_QUERY_SCHEMA, + "query": query_text, + "matched_nodes": matches, + "impacted_artifacts": impacted, + "status": status, + "coverage": {"state": global_state, "path_states": sorted(path_states), "unknown": [] if global_state == "complete" and not truncated else ["lineage_completeness"], "truncated": sorted(set(truncated))}, + "claims": {"impact_status": impact_status, "not_affected": False, "negative_result_is_not_evidence": True}, + } + unsigned["query_digest"] = _sl_sha(unsigned) + return unsigned + + +def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any]: + required = {"schema_version", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} + if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: + raise SBOMLineageError("unsupported query schema") + supplied = query_result.get("query_digest") + unsigned = dict(query_result) + unsigned.pop("query_digest", None) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + raise SBOMLineageError("query digest mismatch") + query = _sl_strict_text(query_result.get("query"), "query", limit=512) + assert query is not None + matched_nodes = _sl_string_list(query_result.get("matched_nodes"), "matched_nodes", maximum=_SL_MAX_QUERY_MATCHES) + for node_id in matched_nodes: + _sl_id(node_id, "matched_node") + impacted = _sl_list(query_result.get("impacted_artifacts"), "impacted_artifacts") + if len(impacted) > _SL_MAX_QUERY_RESULTS: + raise SBOMLineageError("impacted_artifacts exceeds its bound") + impacted_ids: list[str] = [] + path_states: list[str] = [] + for item in impacted: + _sl_require_keys(item, {"artifact_id", "path", "coverage", "confidence", "evidence_refs"}, set(), "impacted_artifact") + artifact_id = _sl_strict_text(item.get("artifact_id"), "artifact_id", limit=256) + assert artifact_id is not None + _sl_id(artifact_id, "artifact_id") + if _sl_node_kind(artifact_id) != "artifact": + raise SBOMLineageError("impacted artifact ID is not an artifact") + if impacted_ids and artifact_id <= impacted_ids[-1]: + raise SBOMLineageError("impacted_artifacts must be unique and sorted") + impacted_ids.append(artifact_id) + path = _sl_list(item.get("path"), "impacted_artifact.path") + if not path or len(path) > _SL_MAX_PATH_LENGTH: + raise SBOMLineageError("impacted_artifact.path is outside its bounds") + checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] + if not any(set((edge["from"], edge["to"])) & set(matched_nodes) for edge in checked_path[:1]): + raise SBOMLineageError("query path is not bound to a matched node") + for first, second in zip(checked_path, checked_path[1:]): + if not set((first["from"], first["to"])) & set((second["from"], second["to"])): + raise SBOMLineageError("query path edges are not endpoint-bound") + if artifact_id not in {checked_path[-1]["from"], checked_path[-1]["to"]}: + raise SBOMLineageError("query path does not terminate at its artifact") + coverage = item.get("coverage") + confidence = item.get("confidence") + if coverage not in _SL_COVERAGE or confidence not in _SL_CONFIDENCE: + raise SBOMLineageError("query path result confidence or coverage is invalid") + expected_path_state = _sl_path_state(checked_path) + expected_path_confidence = _sl_path_confidence(checked_path) + if coverage != expected_path_state or confidence != expected_path_confidence: + raise SBOMLineageError("query path summary is inconsistent") + evidence_refs = _sl_string_list(item.get("evidence_refs"), "impacted_artifact.evidence_refs", maximum=_SL_MAX_EVIDENCE_REFS) + expected_refs = sorted({ref for edge in checked_path for ref in edge.get("evidence_refs", [])}) + if evidence_refs != expected_refs: + raise SBOMLineageError("query evidence references are not bound to its path") + path_states.append(coverage) + coverage = query_result.get("coverage") + _sl_require_keys(coverage, {"state", "path_states", "unknown", "truncated"}, set(), "query.coverage") + coverage_state = coverage.get("state") + if coverage_state not in _SL_COVERAGE: + raise SBOMLineageError("query.coverage.state is invalid") + declared_path_states = _sl_string_list(coverage.get("path_states"), "query.coverage.path_states", maximum=3) + if declared_path_states != sorted(set(path_states)): + raise SBOMLineageError("query coverage path states are inconsistent") + _sl_string_list(coverage.get("unknown"), "query.coverage.unknown", maximum=8) + truncated = _sl_string_list(coverage.get("truncated"), "query.coverage.truncated", maximum=16) + if truncated and coverage_state == "complete": + raise SBOMLineageError("query coverage cannot be complete when truncated") + expected_unknown = [] if coverage_state == "complete" and not truncated else ["lineage_completeness"] + if coverage.get("unknown") != expected_unknown: + raise SBOMLineageError("query coverage unknown state is inconsistent") + status = query_result.get("status") + if status not in {"complete", "partial", "unknown", "not_found"}: + raise SBOMLineageError("query status is invalid") + if impacted: + expected_status = "unknown" if "unknown" in path_states or coverage_state == "unknown" else "partial" if "partial" in path_states or coverage_state == "partial" else "complete" + expected_impact = "not_established" if truncated or "unknown" in path_states else "partial" if "partial" in path_states else "established" + else: + expected_status = "unknown" if coverage_state != "complete" else "not_found" + expected_impact = "not_established" + claims = query_result.get("claims") + _sl_require_keys(claims, {"impact_status", "not_affected", "negative_result_is_not_evidence"}, set(), "query.claims") + if claims.get("impact_status") != expected_impact or claims.get("not_affected") is not False or claims.get("negative_result_is_not_evidence") is not True: + raise SBOMLineageError("query claims are inconsistent with coverage") + if status != expected_status: + raise SBOMLineageError("query status is inconsistent with coverage") + return dict(query_result) + + +def verify_sbom_lineage_query(query_result: Mapping[str, Any]) -> dict[str, Any]: + try: + checked = _sl_validate_query_result(query_result) + except (SBOMLineageError, TypeError, ValueError) as exc: + return {"valid": False, "error": str(exc)} + return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} + + +def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: + """CLI adapter for local SBOM normalization, merge, and query operations.""" + try: + command = getattr(args, "sbom_command", None) + output = getattr(args, "output", None) + if command == "ingest": + document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) + elif command == "merge": + documents = [] + for path in args.documents: + raw_bytes = _sl_read_bounded(Path(path)) + try: + payload = json.loads(raw_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + payload = None + documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) + edges = json.loads(_sl_read_bounded(Path(args.edges)).decode("utf-8")) if getattr(args, "edges", None) else [] + document = build_sbom_lineage(documents, edges=edges) + elif command == "query": + lineage = json.loads(_sl_read_bounded(Path(args.lineage)).decode("utf-8")) + document = query_sbom_lineage(lineage, args.component, limit=getattr(args, "limit", 32)) + else: + raise SBOMLineageError("command must be ingest, merge, or query") + serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" + if output: + Path(output).write_text(serialized, encoding="utf-8") + if getattr(args, "json", False) or not output: + print(serialized, end="") + else: + digest_key = "lineage_digest" if "lineage_digest" in document else "query_digest" if "query_digest" in document else "ingestion_digest" + print(f"sbom {command} -> {output}\n{digest_key}: {document[digest_key]}") + return 0 + except (OSError, TypeError, ValueError, SBOMLineageError) as exc: + print(f"sbom: {exc}") + return 1 # ───────────────────── Prompt-size forensics (#606) ────────────────────────── # # `perseus prompt-size` / `@budget` — byte-accurate, network-free breakdown of @@ -45280,6 +46889,26 @@ def main(): p_artifact.add_argument("--output", "-o", default=None, help="Write the artifact to a file") p_artifact.add_argument("--json", action="store_true", help="Write JSON to stdout even when --output is used") + # sbom/lineage (#995) — offline software supply-chain ingestion and query + p_sbom = sub.add_parser("sbom", help="Ingest SPDX/CycloneDX documents and query software lineage") + sbom_sub = p_sbom.add_subparsers(dest="sbom_command", required=True) + p_sbom_ingest = sbom_sub.add_parser("ingest", help="Normalize one SPDX or CycloneDX JSON/XML document") + p_sbom_ingest.add_argument("document", help="SBOM document path") + p_sbom_ingest.add_argument("--source-ref", default="", dest="source_ref", help="Visibility-safe source reference") + p_sbom_ingest.add_argument("--output", "-o", default=None, help="Write the normalized document to a JSON file") + p_sbom_ingest.add_argument("--json", action="store_true", help="Print machine-readable JSON") + p_sbom_merge = sbom_sub.add_parser("merge", help="Build a queryable lineage graph from normalized SBOM documents") + p_sbom_merge.add_argument("documents", nargs="+", help="SBOM document paths") + p_sbom_merge.add_argument("--edges", default=None, help="Optional JSON file of source/build/artifact/deployment edges") + p_sbom_merge.add_argument("--output", "-o", default=None, help="Write the lineage graph to a JSON file") + p_sbom_merge.add_argument("--json", action="store_true", help="Print machine-readable JSON") + p_sbom_query = sbom_sub.add_parser("query", help="Query a lineage graph for an impacted component/artifact") + p_sbom_query.add_argument("lineage", help="Lineage graph JSON path") + p_sbom_query.add_argument("component", help="Component name, version, purl, or vulnerability reference") + p_sbom_query.add_argument("--limit", type=int, default=32, help="Maximum impacted artifacts") + p_sbom_query.add_argument("--output", "-o", default=None, help="Write the query result to a JSON file") + p_sbom_query.add_argument("--json", action="store_true", help="Print machine-readable JSON") + # memory-efficiency (#929) — deterministic citation-ready telemetry artifact p_memory_efficiency = sub.add_parser("memory-efficiency", help="Emit the offline Vault memory-injection efficiency report") p_memory_efficiency.add_argument("--output", "-o", default=None, help="Write the JSON report to a file") @@ -45897,6 +47526,8 @@ def _add_guide_subcommands(guide_sub): return cmd_code_map(args, cfg) elif args.command == "context-artifact": return cmd_context_artifact(args, cfg) + elif args.command == "sbom": + return cmd_sbom(args, cfg) elif args.command == "memory-efficiency": return cmd_memory_efficiency(args, cfg) elif args.command == "watch": diff --git a/scripts/build.py b/scripts/build.py index 1e8c99f4..e40ac338 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -105,6 +105,7 @@ "src/perseus/code_graph.py", # ← #921: optional symbol/dependency-aware context provider + "src/perseus/sbom_lineage.py", # ← #995: SPDX/CycloneDX software lineage "src/perseus/promptsize.py", # ← #606: perseus prompt-size + @budget forensics (depends on renderer, compress, serve helpers) "src/perseus/cli.py", # includes _bind_registry() call before dispatch ] diff --git a/src/perseus/cli.py b/src/perseus/cli.py index ad83cca7..bb50b781 100644 --- a/src/perseus/cli.py +++ b/src/perseus/cli.py @@ -152,6 +152,26 @@ def main(): p_artifact.add_argument("--output", "-o", default=None, help="Write the artifact to a file") p_artifact.add_argument("--json", action="store_true", help="Write JSON to stdout even when --output is used") + # sbom/lineage (#995) — offline software supply-chain ingestion and query + p_sbom = sub.add_parser("sbom", help="Ingest SPDX/CycloneDX documents and query software lineage") + sbom_sub = p_sbom.add_subparsers(dest="sbom_command", required=True) + p_sbom_ingest = sbom_sub.add_parser("ingest", help="Normalize one SPDX or CycloneDX JSON/XML document") + p_sbom_ingest.add_argument("document", help="SBOM document path") + p_sbom_ingest.add_argument("--source-ref", default="", dest="source_ref", help="Visibility-safe source reference") + p_sbom_ingest.add_argument("--output", "-o", default=None, help="Write the normalized document to a JSON file") + p_sbom_ingest.add_argument("--json", action="store_true", help="Print machine-readable JSON") + p_sbom_merge = sbom_sub.add_parser("merge", help="Build a queryable lineage graph from normalized SBOM documents") + p_sbom_merge.add_argument("documents", nargs="+", help="SBOM document paths") + p_sbom_merge.add_argument("--edges", default=None, help="Optional JSON file of source/build/artifact/deployment edges") + p_sbom_merge.add_argument("--output", "-o", default=None, help="Write the lineage graph to a JSON file") + p_sbom_merge.add_argument("--json", action="store_true", help="Print machine-readable JSON") + p_sbom_query = sbom_sub.add_parser("query", help="Query a lineage graph for an impacted component/artifact") + p_sbom_query.add_argument("lineage", help="Lineage graph JSON path") + p_sbom_query.add_argument("component", help="Component name, version, purl, or vulnerability reference") + p_sbom_query.add_argument("--limit", type=int, default=32, help="Maximum impacted artifacts") + p_sbom_query.add_argument("--output", "-o", default=None, help="Write the query result to a JSON file") + p_sbom_query.add_argument("--json", action="store_true", help="Print machine-readable JSON") + # memory-efficiency (#929) — deterministic citation-ready telemetry artifact p_memory_efficiency = sub.add_parser("memory-efficiency", help="Emit the offline Vault memory-injection efficiency report") p_memory_efficiency.add_argument("--output", "-o", default=None, help="Write the JSON report to a file") @@ -769,6 +789,8 @@ def _add_guide_subcommands(guide_sub): return cmd_code_map(args, cfg) elif args.command == "context-artifact": return cmd_context_artifact(args, cfg) + elif args.command == "sbom": + return cmd_sbom(args, cfg) elif args.command == "memory-efficiency": return cmd_memory_efficiency(args, cfg) elif args.command == "watch": diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py new file mode 100644 index 00000000..2f02ec77 --- /dev/null +++ b/src/perseus/sbom_lineage.py @@ -0,0 +1,1610 @@ +"""Offline SPDX/CycloneDX ingestion and queryable software lineage (#995). + +The module deliberately stays stdlib-only. It normalizes the two common SBOM +families into a bounded, digest-sealed projection and keeps coverage/unknown +states visible. It does not replace a generator, scanner, VEX authority, or +signing system: references are carried through only when the input supplies +those references. +""" +from __future__ import annotations + +import hashlib +import json +import re +import xml.etree.ElementTree as _sl_et +from collections import deque +from pathlib import Path +from typing import Any, Mapping + +_SL_SCHEMA = "perseus-sbom/v1" +_SL_LINEAGE_SCHEMA = "perseus-software-lineage/v1" +_SL_QUERY_SCHEMA = "perseus-software-lineage-query/v1" +_SL_FORMATS = frozenset({"SPDX", "CycloneDX"}) +_SL_SPDX_VERSIONS = frozenset({"2.2", "2.3"}) +_SL_CDX_VERSIONS = frozenset({"1.4", "1.5", "1.6"}) +_SL_CONFIDENCE = frozenset({"high", "medium", "low", "unknown"}) +_SL_COVERAGE = frozenset({"complete", "partial", "unknown"}) +_SL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") +_SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") +_SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") +_SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") +_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer\s+|basic\s+|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*=|credential\s*=)") +_SL_REFERENCE_TYPES = frozenset({ + "advisory", "attestation", "cve", "distribution", "documentation", "license", + "purl", "signature", "vex", "vulnerability", "website", "other", +}) +_SL_FORBIDDEN_MARKERS = frozenset({"api_key", "authorization", "password", "secret", "token", "credential"}) +_SL_MAX_INPUT_BYTES = 8 * 1024 * 1024 +_SL_MAX_XML_ELEMENTS = 20_000 +_SL_MAX_XML_DEPTH = 128 +_SL_MAX_COMPONENTS = 512 +_SL_MAX_RELATIONSHIPS = 1024 +_SL_MAX_DOCUMENTS = 64 +_SL_MAX_EDGES = 4096 +_SL_MAX_REFERENCES = 64 +_SL_MAX_PROPERTIES = 64 +_SL_MAX_IDENTIFIERS = 64 +_SL_MAX_LICENSES = 32 +_SL_MAX_HASHES = 8 +_SL_MAX_EVIDENCE_REFS = 64 +_SL_MAX_DEPENDENCY_TARGETS = 128 +_SL_MAX_QUERY_MATCHES = 256 +_SL_MAX_QUERY_RESULTS = 256 +_SL_MAX_PATH_LENGTH = 32 +_SL_MAX_NODES = 20_000 + + +class SBOMLineageError(ValueError): + """Raised when an SBOM or lineage projection cannot be verified.""" + + +def _sl_json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) + + +def _sl_sha(value: Any) -> str: + return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() + + +def _sl_sensitive(value: str) -> bool: + """Return whether a scalar looks like it carries a credential.""" + authority = value.split("://", 1)[1].split("/", 1)[0] if "://" in value else "" + return bool(_SL_SENSITIVE_REFERENCE_RE.search(value) or (authority and "@" in authority)) + + +def _sl_truncated(truncated: list[str] | None, name: str) -> None: + if truncated is not None and name not in truncated: + truncated.append(name) + + +def _sl_list(value: Any, field: str) -> list[Any]: + if value is None: + return [] + if not isinstance(value, list): + raise SBOMLineageError(f"{field} must be a list") + return value + + +def _sl_nonnegative_int(value: Any, field: str, *, maximum: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise SBOMLineageError(f"{field} must be an integer") + if value < 0 or (maximum is not None and value > maximum): + bound = f" between 0 and {maximum}" if maximum is not None else "" + raise SBOMLineageError(f"{field} must be{bound}") + return value + + +def _sl_limit(value: Any, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not 1 <= value <= _SL_MAX_QUERY_RESULTS: + raise SBOMLineageError(f"{field} must be between 1 and {_SL_MAX_QUERY_RESULTS}") + return value + + +def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512) -> str: + if value is None: + if required: + raise SBOMLineageError(f"{field} is required") + return "" + if not isinstance(value, str): + value = str(value) + text = re.sub(r"[\x00-\x1f\x7f]", " ", value).strip() + if required and not text: + raise SBOMLineageError(f"{field} is required") + if len(text) > limit: + raise SBOMLineageError(f"{field} exceeds {limit} characters") + return text + + +def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: + text = _sl_text(value if value is not None else fallback, field, required=True, limit=256) + if not _SL_ID_RE.fullmatch(text): + raise SBOMLineageError(f"{field} is not a bounded identifier") + return text + + +def _sl_safe_locator(value: Any, field: str) -> str: + """Keep public identifiers; hash credential-bearing scalar values.""" + text = _sl_text(value, field, required=True, limit=1024) + if _sl_sensitive(text): + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + return text + + +def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: + if value in (None, ""): + return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" + text = _sl_text(value, "source_ref", required=True, limit=512) + if text.startswith("sha256:") and re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text): + return text.lower() + if _SL_PUBLIC_SOURCE_RE.fullmatch(text) and not _SL_SENSITIVE_REFERENCE_RE.search(text): + return text + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sl_strict_text(value: Any, field: str, *, allow_none: bool = False, limit: int = 512) -> str | None: + if value is None and allow_none: + return None + if not isinstance(value, str): + raise SBOMLineageError(f"{field} must be a string") + if value != value.strip() or re.search(r"[\x00-\x1f\x7f]", value): + raise SBOMLineageError(f"{field} contains invalid characters") + if not value or len(value) > limit: + raise SBOMLineageError(f"{field} is empty or exceeds {limit} characters") + if _sl_sensitive(value): + raise SBOMLineageError(f"{field} contains unsafe credential material") + return value + + +def _sl_strict_safe_locator(value: Any, field: str, *, limit: int = 1024) -> str: + text = _sl_strict_text(value, field, limit=limit) + assert text is not None + if _sl_safe_locator(text, field) != text: + raise SBOMLineageError(f"{field} contains unsafe credential material") + return text + + +def _sl_strict_source_ref(value: Any) -> str: + text = _sl_strict_text(value, "source_ref", limit=512) + assert text is not None + if text.startswith("sha256:"): + if not re.fullmatch(r"sha256:[0-9a-f]{64}", text): + raise SBOMLineageError("source_ref digest is malformed") + return text + if not _SL_PUBLIC_SOURCE_RE.fullmatch(text) or _sl_sensitive(text): + raise SBOMLineageError("source_ref is not a visibility-safe reference") + return text + + +def _sl_safe_hashes(value: Any, *, truncated: list[str] | None = None) -> list[dict[str, str]]: + if value is None: + return [] + hashes = _sl_list(value, "hashes") + if len(hashes) > _SL_MAX_HASHES: + _sl_truncated(truncated, "hashes") + result: list[dict[str, str]] = [] + for raw in hashes[:_SL_MAX_HASHES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("hashes must contain objects") + algorithm = _sl_text(raw.get("alg"), "hash_algorithm", required=True, limit=32) + content = _sl_text(raw.get("content"), "hash", required=True, limit=256) + # Only retain actual digest-looking values. Arbitrary hash content is + # an attacker-controlled data channel and is deliberately dropped. + if re.fullmatch(r"[0-9a-fA-F]{32,128}", content): + result.append({"alg": algorithm, "content": content.lower()}) + return result + + +def _sl_local(element: Any) -> str: + return str(getattr(element, "tag", "")).rsplit("}", 1)[-1] + + +def _sl_child(element: Any, name: str) -> Any | None: + for child in list(element): + if _sl_local(child) == name: + return child + return None + + +def _sl_children(element: Any, name: str) -> list[Any]: + return [child for child in list(element) if _sl_local(child) == name] + + +def _sl_xml_text(element: Any, name: str, *, default: str = "") -> str: + if element is None: + return default + child = _sl_child(element, name) + return (child.text or "").strip() if child is not None and child.text else default + + +def _sl_validate_xml_tree(root: Any) -> None: + count = 0 + stack = [(root, 1)] + while stack: + element, depth = stack.pop() + count += 1 + if count > _SL_MAX_XML_ELEMENTS: + raise SBOMLineageError("SBOM XML contains too many elements") + if depth > _SL_MAX_XML_DEPTH: + raise SBOMLineageError("SBOM XML nesting is too deep") + stack.extend((child, depth + 1) for child in list(element)) + + +def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: + wanted = {name.casefold() for name in names} + for child in list(element) if element is not None else []: + if _sl_local(child).casefold() not in wanted: + continue + text = (child.text or "").strip() if child.text else "" + if text: + return text + for key, value in child.attrib.items(): + if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: + return str(value).lstrip("#") + return default + + +def _sl_descendants(root: Any, *names: str) -> list[Any]: + wanted = {name.casefold() for name in names} + return [element for element in root.iter() if _sl_local(element).casefold() in wanted] + + +def _sl_supplier(value: Any) -> str: + if isinstance(value, Mapping): + return _sl_text(value.get("name"), "supplier") + if isinstance(value, list): + names = [_sl_supplier(item) for item in value] + return "; ".join(item for item in names if item) + return _sl_text(value, "supplier") + + +def _sl_normalize_reference(reference_type: Any, locator: Any, *, category: Any = "", comment: Any = None, hashes: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: + ref_type = _sl_text(reference_type, "reference_type", required=True, limit=64).casefold() + locator_text = _sl_safe_locator(locator, "reference_locator") + result: dict[str, Any] = { + "type": ref_type if ref_type in _SL_REFERENCE_TYPES else "other", + "locator": locator_text, + } + category_text = _sl_text(category, "reference_category", limit=64) + if category_text: + result["category"] = category_text + safe_hashes = _sl_safe_hashes(hashes, truncated=truncated) + if safe_hashes: + result["hashes"] = safe_hashes + return result + + +def _sl_spdx_references(value: Any, *, truncated: list[str] | None = None) -> tuple[list[dict[str, Any]], list[str]]: + references: list[dict[str, Any]] = [] + identifiers: list[str] = [] + if value is None: + return references, identifiers + refs = _sl_list(value, "SPDX externalRefs") + if len(refs) > _SL_MAX_REFERENCES: + _sl_truncated(truncated, "externalRefs") + for raw in refs[:_SL_MAX_REFERENCES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("SPDX externalRefs must contain objects") + locator = raw.get("referenceLocator") + ref_type = raw.get("referenceType", "other") + if locator is None: + raise SBOMLineageError("SPDX externalRef requires referenceLocator") + item = _sl_normalize_reference( + ref_type, locator, category=raw.get("referenceCategory"), + hashes=raw.get("hashes"), truncated=truncated, + ) + references.append(item) + if str(ref_type).casefold() == "purl": + identifiers.append(item["locator"]) + return references, identifiers + + +def _sl_cdx_references(value: Any, *, truncated: list[str] | None = None) -> tuple[list[dict[str, Any]], list[str]]: + references: list[dict[str, Any]] = [] + identifiers: list[str] = [] + if value is None: + return references, identifiers + refs = _sl_list(value, "CycloneDX externalReferences") + if len(refs) > _SL_MAX_REFERENCES: + _sl_truncated(truncated, "externalReferences") + for raw in refs[:_SL_MAX_REFERENCES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("CycloneDX externalReferences must contain objects") + locator = raw.get("url") + if not locator: + raise SBOMLineageError("CycloneDX externalReference requires url") + if "hashes" in raw and raw.get("hashes") is not None and not isinstance(raw.get("hashes"), list): + raise SBOMLineageError("CycloneDX externalReference hashes must be a list") + item = _sl_normalize_reference( + raw.get("type", "other"), locator, hashes=raw.get("hashes"), truncated=truncated, + ) + references.append(item) + if str(raw.get("type", "")).casefold() == "purl": + identifiers.append(item["locator"]) + return references, identifiers + + +def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + if value is None: + return result + properties = _sl_list(value, "properties") + if len(properties) > _SL_MAX_PROPERTIES: + _sl_truncated(truncated, "properties") + for raw in properties[:_SL_MAX_PROPERTIES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("properties must contain objects") + name = _sl_text(raw.get("name"), "property_name", limit=128) + prop_value = _sl_text(raw.get("value"), "property_value", required=True, limit=1024) + if name and prop_value and any(marker in name.casefold() for marker in ("vex", "vuln", "signature", "attestation")): + result.append({ + "type": name, + "locator": "sha256:" + hashlib.sha256(prop_value.encode("utf-8")).hexdigest(), + }) + return result + + +def _sl_component( + *, + component_id: Any, + name: Any, + version: Any = None, + supplier: Any = None, + identifiers: Any = None, + references: Any = None, + component_type: Any = None, + licenses: Any = None, + truncated: list[str] | None = None, +) -> dict[str, Any]: + normalized_name = _sl_text(name, "component_name", required=True, limit=256) + normalized_version = _sl_text(version, "component_version", limit=128) + normalized_id = _sl_id(component_id, "component ID") + ids = {normalized_id} + component_truncated: list[str] = list(truncated or []) + if identifiers is not None and not isinstance(identifiers, (list, tuple, set)): + raise SBOMLineageError("component identifiers must be a list") + identifier_values = list(identifiers or []) + if len(identifier_values) > _SL_MAX_IDENTIFIERS: + _sl_truncated(component_truncated, "identifiers") + for identifier in identifier_values[:_SL_MAX_IDENTIFIERS]: + text = _sl_safe_locator(identifier, "component_identifier") + if text: + ids.add(text) + safe_references = [] + if references is not None and not isinstance(references, (list, tuple)): + raise SBOMLineageError("component references must be a list") + reference_values = list(references or []) + if len(reference_values) > _SL_MAX_REFERENCES: + _sl_truncated(component_truncated, "references") + for item in reference_values[:_SL_MAX_REFERENCES]: + if not isinstance(item, Mapping): + raise SBOMLineageError("component references must contain objects") + safe_references.append(dict(item)) + license_values: list[str] = [] + if licenses is None: + license_values_input: list[Any] = [] + elif isinstance(licenses, (list, tuple, set)): + license_values_input = list(licenses) + else: + license_values_input = [licenses] + if len(license_values_input) > _SL_MAX_LICENSES: + _sl_truncated(component_truncated, "licenses") + if licenses is not None: + for item in license_values_input[:_SL_MAX_LICENSES]: + if isinstance(item, Mapping): + item = item.get("license", item.get("id")) + if isinstance(item, Mapping): + item = item.get("id") or item.get("name") + text = _sl_text(item, "license", limit=128) + if text: + license_values.append(text) + elif licenses: + text = _sl_text(licenses, "license", limit=128) + if text: + license_values.append(text) + unknown = [] + if not normalized_version: + unknown.append("version") + if not _sl_supplier(supplier): + unknown.append("supplier") + if component_truncated: + unknown.append("truncated:" + ",".join(sorted(set(component_truncated)))) + coverage_state = "complete" if normalized_version and _sl_supplier(supplier) and not component_truncated else "partial" + return { + "component_id": normalized_id, + "name": normalized_name, + "version": normalized_version or None, + "supplier": _sl_supplier(supplier) or None, + "identifiers": sorted(ids), + "references": sorted(safe_references, key=lambda item: (item.get("type", ""), item.get("locator", ""))), + "licenses": sorted(set(license_values)), + "component_type": _sl_text(component_type, "component_type", limit=64) or "unknown", + "coverage": {"state": coverage_state, "unknown": sorted(set(unknown)), "truncated": sorted(set(component_truncated))}, + } + + +def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: + source_id = _sl_id(source, "relationship.from") + target_id = _sl_id(target, "relationship.to") + rel_type = _sl_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") + if not isinstance(confidence, str) or confidence not in _SL_CONFIDENCE: + raise SBOMLineageError("relationship confidence is unsupported") + if not isinstance(coverage, str) or coverage not in _SL_COVERAGE: + raise SBOMLineageError("relationship coverage is unsupported") + refs = [] + if evidence_refs is not None and not isinstance(evidence_refs, (list, tuple)): + raise SBOMLineageError("relationship evidence_refs must be a list") + evidence_values = list(evidence_refs or []) + if len(evidence_values) > _SL_MAX_EVIDENCE_REFS: + _sl_truncated(truncated, "evidence_refs") + refs = sorted({_sl_id(ref, "relationship.evidence_ref") for ref in evidence_values[:_SL_MAX_EVIDENCE_REFS]}) + result: dict[str, Any] = { + "from": source_id, + "to": target_id, + "type": rel_type, + "confidence": confidence, + "coverage": coverage, + } + if refs: + result["evidence_refs"] = refs + return result + + +def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: + component_truncated: list[str] = [] + references, identifiers = _sl_spdx_references(raw.get("externalRefs"), truncated=component_truncated) + if truncated is not None: + truncated.extend(item for item in component_truncated if item not in truncated) + return _sl_component( + component_id=raw.get("SPDXID"), + name=raw.get("name"), + version=raw.get("versionInfo"), + supplier=raw.get("supplier"), + identifiers=identifiers, + references=references, + component_type="package", + licenses=raw.get("licenseConcluded"), + truncated=component_truncated, + ) + + +def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: + component_truncated: list[str] = [] + references, identifiers = _sl_cdx_references(raw.get("externalReferences"), truncated=component_truncated) + if "purl" in raw and raw.get("purl") is not None: + identifiers.append(_sl_safe_locator(raw.get("purl"), "purl")) + references.extend(_sl_properties(raw.get("properties"), truncated=component_truncated)) + if truncated is not None: + truncated.extend(item for item in component_truncated if item not in truncated) + return _sl_component( + component_id=raw.get("bom-ref"), + name=raw.get("name"), + version=raw.get("version"), + supplier=raw.get("supplier"), + identifiers=identifiers, + references=references, + component_type=raw.get("type"), + licenses=raw.get("licenses"), + truncated=component_truncated, + ) + + +def _sl_validate_spdx_version(value: Any) -> str: + version_text = _sl_text(value, "SPDX version", required=True, limit=32) + match = _SL_SPDX_VERSION_RE.fullmatch(version_text) + if not match or match.group(1) not in _SL_SPDX_VERSIONS: + raise SBOMLineageError(f"unsupported SPDX version: {version_text}") + return match.group(1) + + +def _sl_validate_cdx_version(value: Any) -> str: + version_text = _sl_text(value, "CycloneDX version", required=True, limit=32) + if version_text not in _SL_CDX_VERSIONS: + raise SBOMLineageError(f"unsupported CycloneDX version: {version_text}") + return version_text + + +def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: + if fmt not in _SL_FORMATS: + raise SBOMLineageError("unsupported SBOM format") + component_ids: set[str] = set() + for item in components: + component_id = item.get("component_id") if isinstance(item, Mapping) else None + if not isinstance(component_id, str) or component_id in component_ids: + raise SBOMLineageError("duplicate or missing component ID") + component_ids.add(component_id) + known_ids = set(component_ids) + if fmt == "SPDX": + known_ids.add(document_id) + dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) + truncation = sorted(set(truncated or [])) + unknown = [] + if not components: + unknown.append("components") + if not relationships: + unknown.append("relationships") + if dangling: + unknown.append("dangling_relationships") + if truncation: + unknown.append("truncated:" + ",".join(truncation)) + if any(item.get("coverage", {}).get("state") != "complete" for item in components): + unknown.append("component_metadata") + if not components: + coverage_state = "unknown" + elif dangling or truncation or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: + coverage_state = "partial" + else: + coverage_state = "complete" + unsigned = { + "schema_version": _SL_SCHEMA, + "format": fmt, + "spec_version": spec_version, + "document_id": document_id, + "document_name": document_name or None, + "document_sha256": hashlib.sha256(raw_bytes).hexdigest(), + "source_ref": source_ref, + "created_at": created_at or None, + "supplier": supplier or None, + "components": sorted(components, key=lambda item: (item["component_id"] == metadata_component_id, item["component_id"])), + "relationships": sorted(relationships, key=lambda item: (item["from"], item["to"], item["type"])), + "coverage": { + "state": coverage_state, + "unknown": sorted(set(unknown)), + "component_count": len(components), + "relationship_count": len(relationships), + "truncated": truncation, + "dangling_relationships": dangling, + }, + } + unsigned["ingestion_digest"] = _sl_sha(unsigned) + return unsigned + + +def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + version = _sl_validate_spdx_version(value.get("spdxVersion")) + document_id = _sl_id(value.get("SPDXID"), "SPDXID") + creation = value.get("creationInfo", {}) + if creation is None: + creation = {} + if not isinstance(creation, Mapping): + raise SBOMLineageError("creationInfo must be an object") + creators = creation.get("creators", []) + if not isinstance(creators, list): + raise SBOMLineageError("creationInfo.creators must be a list") + supplier = _sl_supplier(creators[0] if creators else "") + packages = _sl_list(value.get("packages"), "packages") + raw_relationships = _sl_list(value.get("relationships"), "relationships") + truncated = [] + if len(packages) > _SL_MAX_COMPONENTS: + truncated.append("packages") + if len(raw_relationships) > _SL_MAX_RELATIONSHIPS: + truncated.append("relationships") + components = [] + for item in packages[:_SL_MAX_COMPONENTS]: + if not isinstance(item, Mapping): + raise SBOMLineageError("packages must contain objects") + components.append(_sl_spdx_component(item, truncated=truncated)) + relationships = [] + for raw in raw_relationships[:_SL_MAX_RELATIONSHIPS]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("relationships must contain objects") + relationships.append(_sl_relationship( + raw.get("spdxElementId"), raw.get("relatedSpdxElement"), + raw.get("relationshipType", "related"), truncated=truncated, + )) + return _sl_finalize_document( + fmt="SPDX", spec_version=version, document_id=document_id, + document_name=_sl_text(value.get("name"), "document_name"), + created_at=_sl_text(creation.get("created"), "created_at"), supplier=supplier, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + ) + + +def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + if value.get("bomFormat") != "CycloneDX": + raise SBOMLineageError("unsupported SBOM format") + version = _sl_validate_cdx_version(value.get("specVersion")) + metadata = value.get("metadata", {}) + if metadata is None: + metadata = {} + if not isinstance(metadata, Mapping): + raise SBOMLineageError("metadata must be an object") + metadata_component = metadata.get("component") + if metadata_component is not None and not isinstance(metadata_component, Mapping): + raise SBOMLineageError("metadata.component must be an object") + raw_components = [] + truncated = [] + if metadata_component: + raw_components.append(metadata_component) + raw_component_list = _sl_list(value.get("components"), "components") + raw_dependency_list = _sl_list(value.get("dependencies"), "dependencies") + if len(raw_component_list) > _SL_MAX_COMPONENTS: + truncated.append("components") + if len(raw_dependency_list) > _SL_MAX_RELATIONSHIPS: + truncated.append("dependencies") + for item in raw_component_list[:_SL_MAX_COMPONENTS]: + if not isinstance(item, Mapping): + raise SBOMLineageError("components must contain objects") + raw_components.append(item) + components: list[dict[str, Any]] = [] + for raw in raw_components: + components.append(_sl_cdx_component(raw, truncated=truncated)) + relationships = [] + for raw in raw_dependency_list[:_SL_MAX_RELATIONSHIPS]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("dependencies must contain objects") + source = raw.get("ref") + targets = _sl_list(raw.get("dependsOn"), "dependsOn") + if len(targets) > _SL_MAX_DEPENDENCY_TARGETS: + truncated.append("dependency_edges") + for target in targets[:_SL_MAX_DEPENDENCY_TARGETS]: + relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated)) + creators = metadata.get("authors", []) + if not isinstance(creators, list): + raise SBOMLineageError("metadata.authors must be a list") + supplier = _sl_supplier(creators[0] if creators else (metadata_component or {})) + document_id = _sl_id(value.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + return _sl_finalize_document( + fmt="CycloneDX", spec_version=version, document_id=document_id, + document_name="CycloneDX BOM", created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + metadata_component_id=components[0]["component_id"] if metadata_component else "", + ) + + +def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + documents = _sl_descendants(root, "SpdxDocument") + document = documents[0] if documents else root + version = _sl_validate_spdx_version(_sl_xml_value(document, "spdxVersion")) + raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") + if not raw_document_id: + raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") + document_id = _sl_id(raw_document_id, "SPDXID") + creation = next(iter(_sl_descendants(document, "creationInfo")), None) + created_at = _sl_xml_value(creation, "created") + creator = _sl_xml_value(creation, "creator") + packages = _sl_descendants(document, "Package", "package") + relationships_raw = _sl_descendants(document, "Relationship", "relationship") + truncated = [] + if len(packages) > _SL_MAX_COMPONENTS: + truncated.append("packages") + if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: + truncated.append("relationships") + components = [] + for raw in packages[:_SL_MAX_COMPONENTS]: + component_truncated: list[str] = [] + refs = [] + identifiers = [] + for ref in _sl_descendants(raw, "ExternalRef", "externalRef"): + locator = _sl_xml_value(ref, "referenceLocator") + if not locator: + raise SBOMLineageError("SPDX externalRef requires referenceLocator") + ref_type = _sl_xml_value(ref, "referenceType", default="other") + item = _sl_normalize_reference( + ref_type, locator, category=_sl_xml_value(ref, "referenceCategory"), + truncated=component_truncated, + ) + refs.append(item) + if ref_type.casefold() == "purl": + identifiers.append(item["locator"]) + truncated.extend(item for item in component_truncated if item not in truncated) + component_id = _sl_xml_value(raw, "SPDXID", "spdxid") + if not component_id: + component_id = next((str(value).lstrip("#") for key, value in raw.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") + components.append(_sl_component( + component_id=component_id, + name=_sl_xml_value(raw, "name"), + version=_sl_xml_value(raw, "versionInfo"), + supplier=_sl_xml_value(raw, "supplier"), + identifiers=identifiers, + references=refs, + component_type="package", + licenses=_sl_xml_value(raw, "licenseConcluded"), + truncated=component_truncated, + )) + relationships = [] + for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: + relationships.append(_sl_relationship( + _sl_xml_value(raw, "spdxElementId"), + _sl_xml_value(raw, "relatedSpdxElement"), + _sl_xml_value(raw, "relationshipType", default="related"), + truncated=truncated, + )) + return _sl_finalize_document( + fmt="SPDX", spec_version=version, document_id=document_id, + document_name=_sl_xml_value(document, "name"), created_at=created_at, supplier=creator, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + ) + + +def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) + document_id = _sl_id(_sl_xml_text(root, "SPDXID"), "SPDXID") + creation = _sl_child(root, "creationInfo") + created_at = _sl_xml_text(creation, "created") if creation is not None else "" + creator = _sl_xml_text(creation, "creator") if creation is not None else "" + packages = _sl_children(root, "package") + relationships_raw = _sl_children(root, "relationship") + truncated: list[str] = [] + if len(packages) > _SL_MAX_COMPONENTS: + truncated.append("packages") + if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: + truncated.append("relationships") + components = [] + for raw in packages[:_SL_MAX_COMPONENTS]: + component_truncated: list[str] = [] + refs = [] + identifiers = [] + for ref in _sl_children(raw, "externalRef"): + locator = _sl_xml_text(ref, "referenceLocator") + ref_type = _sl_xml_text(ref, "referenceType", default="other") + if not locator: + raise SBOMLineageError("SPDX externalRef requires referenceLocator") + item = _sl_normalize_reference( + ref_type, locator, category=_sl_xml_text(ref, "referenceCategory"), + truncated=component_truncated, + ) + refs.append(item) + if ref_type.casefold() == "purl": + identifiers.append(item["locator"]) + truncated.extend(item for item in component_truncated if item not in truncated) + components.append(_sl_component( + component_id=_sl_xml_text(raw, "SPDXID"), name=_sl_xml_text(raw, "name"), + version=_sl_xml_text(raw, "versionInfo"), supplier=_sl_xml_text(raw, "supplier"), + identifiers=identifiers, references=refs, component_type="package", + licenses=_sl_xml_text(raw, "licenseConcluded"), + truncated=component_truncated, + )) + relationships = [] + for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: + relationships.append(_sl_relationship( + _sl_xml_text(raw, "spdxElementId"), _sl_xml_text(raw, "relatedSpdxElement"), + _sl_xml_text(raw, "relationshipType", default="related"), + truncated=truncated, + )) + return _sl_finalize_document( + fmt="SPDX", spec_version=version, document_id=document_id, + document_name=_sl_xml_text(root, "name"), created_at=created_at, supplier=creator, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + ) + + +def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None) -> dict[str, Any]: + component_truncated: list[str] = [] + refs = [] + identifiers = [] + purl = _sl_xml_text(raw, "purl") + if purl: + identifiers.append(_sl_safe_locator(purl, "purl")) + external = _sl_child(raw, "externalReferences") + if external is not None: + reference_nodes = _sl_children(external, "reference") + if len(reference_nodes) > _SL_MAX_REFERENCES: + component_truncated.append("externalReferences") + for ref in reference_nodes[:_SL_MAX_REFERENCES]: + locator = _sl_xml_text(ref, "url") + if not locator: + raise SBOMLineageError("CycloneDX externalReference requires url") + refs.append(_sl_normalize_reference( + _sl_xml_text(ref, "type", default="other"), locator, + truncated=component_truncated, + )) + licenses = [] + license_container = _sl_child(raw, "licenses") + if license_container is not None: + license_nodes = _sl_children(license_container, "license") + if len(license_nodes) > _SL_MAX_LICENSES: + component_truncated.append("licenses") + for item in license_nodes[:_SL_MAX_LICENSES]: + licenses.append(_sl_xml_text(item, "id") or _sl_xml_text(item, "name")) + if truncated is not None: + truncated.extend(item for item in component_truncated if item not in truncated) + return _sl_component( + component_id=raw.attrib.get("bom-ref"), name=_sl_xml_text(raw, "name"), + version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_child(raw, "supplier"), "name"), + identifiers=identifiers, references=refs, component_type=raw.attrib.get("type"), licenses=licenses, + truncated=component_truncated, + ) + + +def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + namespace = root.tag.split("}", 1)[0].lstrip("{") if "}" in root.tag else "" + match = _SL_CDX_NAMESPACE_RE.search(namespace) + if not match: + raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") + version = _sl_validate_cdx_version(match.group(1)) + metadata = _sl_child(root, "metadata") + metadata_component = _sl_child(metadata, "component") if metadata is not None else None + raw_components = [] + truncated: list[str] = [] + if metadata_component is not None: + raw_components.append(metadata_component) + components_node = _sl_child(root, "components") + if components_node is not None: + component_nodes = _sl_children(components_node, "component") + if len(component_nodes) > _SL_MAX_COMPONENTS: + truncated.append("components") + raw_components.extend(component_nodes[:_SL_MAX_COMPONENTS]) + components = [] + for raw in raw_components: + components.append(_sl_cdx_xml_component(raw, truncated=truncated)) + relationships = [] + dependencies_node = _sl_child(root, "dependencies") + if dependencies_node is not None: + dependency_nodes = _sl_children(dependencies_node, "dependency") + if len(dependency_nodes) > _SL_MAX_RELATIONSHIPS: + truncated.append("dependencies") + for dependency in dependency_nodes[:_SL_MAX_RELATIONSHIPS]: + source = dependency.attrib.get("ref") + children = _sl_children(dependency, "dependency") + if len(children) > _SL_MAX_DEPENDENCY_TARGETS: + truncated.append("dependency_edges") + for child in children[:_SL_MAX_DEPENDENCY_TARGETS]: + relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated)) + timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" + authors = _sl_child(metadata, "authors") if metadata is not None else None + author = _sl_xml_text(_sl_child(authors, "author"), "name") if authors is not None else "" + document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + return _sl_finalize_document( + fmt="CycloneDX", spec_version=version, document_id=document_id, + document_name="CycloneDX BOM", created_at=timestamp, supplier=author, + components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, + metadata_component_id=components[0]["component_id"] if metadata_component is not None else "", + ) + + +def _sl_read_bounded(path: Path) -> bytes: + """Read a file only after checking its size, including race re-check.""" + try: + size = path.stat().st_size + except OSError as exc: + raise SBOMLineageError(f"could not stat SBOM: {exc}") from exc + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + try: + raw_bytes = path.read_bytes() + except OSError as exc: + raise SBOMLineageError(f"could not read SBOM: {exc}") from exc + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return raw_bytes + + +def _sl_payload(document: Any) -> tuple[Any, bytes]: + if isinstance(document, Path): + return _sl_payload(_sl_read_bounded(document)) + if isinstance(document, bytes): + raw_bytes = document + elif isinstance(document, bytearray): + raw_bytes = bytes(document) + elif isinstance(document, Mapping): + try: + raw_bytes = _sl_json(document).encode("utf-8") + except (TypeError, ValueError) as exc: + raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc + elif isinstance(document, str): + possible_path = Path(document) + if "\n" not in document and possible_path.exists(): + return _sl_payload(possible_path) + raw_bytes = document.encode("utf-8") + else: + raise SBOMLineageError("SBOM input must be bytes, text, path, or object") + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + if not raw_bytes.strip(): + raise SBOMLineageError("SBOM input is empty") + try: + value = json.loads(raw_bytes.decode("utf-8")) + if not isinstance(value, Mapping): + raise SBOMLineageError("SBOM JSON root must be an object") + return dict(value), raw_bytes + except (UnicodeDecodeError, json.JSONDecodeError): + try: + root = _sl_et.fromstring(raw_bytes) + _sl_validate_xml_tree(root) + except _sl_et.ParseError as exc: + raise SBOMLineageError("SBOM is neither valid JSON nor XML") from exc + return root, raw_bytes + + +def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, Any]: + """Parse one SPDX or CycloneDX JSON/XML document into a safe projection.""" + value, raw_bytes = _sl_payload(document) + normalized_source = _sl_safe_source_ref(source_ref, raw_bytes) + if isinstance(value, Mapping): + if value.get("spdxVersion") is not None: + return _sl_parse_spdx_json(value, raw_bytes, normalized_source) + if value.get("bomFormat") is not None: + return _sl_parse_cdx_json(value, raw_bytes, normalized_source) + raise SBOMLineageError("SBOM format is missing or unsupported") + root_name = _sl_local(value).casefold() + if root_name == "spdxdocument": + if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): + return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) + return _sl_spdx_xml(value, raw_bytes, normalized_source) + if root_name == "rdf" and _sl_descendants(value, "SpdxDocument"): + return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) + if root_name == "bom": + return _sl_parse_cdx_xml(value, raw_bytes, normalized_source) + raise SBOMLineageError("SBOM XML format is missing or unsupported") + + +def _sl_node_kind(node_id: str) -> str: + if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("document:"): + return "document" + if node_id.startswith("artifact:"): + return "artifact" + if node_id.startswith("deployment:"): + return "deployment" + if node_id.startswith("build:"): + return "build" + if node_id.startswith("source:"): + return "source" + return "component" + + +def _sl_require_keys(value: Any, required: set[str], optional: set[str], field: str) -> None: + if not isinstance(value, Mapping) or not required.issubset(set(value)) or set(value) - required - optional: + raise SBOMLineageError(f"{field} schema is invalid") + + +def _sl_string_list(value: Any, field: str, *, maximum: int) -> list[str]: + items = _sl_list(value, field) + if len(items) > maximum: + raise SBOMLineageError(f"{field} exceeds {maximum} items") + result = [] + for item in items: + text = _sl_strict_text(item, field, limit=1024) + assert text is not None + result.append(text) + if result != sorted(set(result)): + raise SBOMLineageError(f"{field} must be unique and sorted") + return result + + +def _sl_validate_reference(reference: Any) -> dict[str, Any]: + _sl_require_keys(reference, {"type", "locator"}, {"category", "hashes"}, "reference") + ref_type = _sl_strict_text(reference.get("type"), "reference.type", limit=64) + assert ref_type is not None + if ref_type not in _SL_REFERENCE_TYPES: + raise SBOMLineageError("reference.type is unsupported") + locator = _sl_strict_safe_locator(reference.get("locator"), "reference.locator") + result: dict[str, Any] = {"type": ref_type, "locator": locator} + if "category" in reference: + category = _sl_strict_text(reference.get("category"), "reference.category", limit=64) + assert category is not None + result["category"] = category + if "hashes" in reference: + hashes = _sl_list(reference.get("hashes"), "reference.hashes") + if not hashes or len(hashes) > _SL_MAX_HASHES: + raise SBOMLineageError("reference.hashes is outside its bounds") + checked_hashes = [] + for item in hashes: + _sl_require_keys(item, {"alg", "content"}, set(), "reference.hash") + algorithm = _sl_strict_text(item.get("alg"), "reference.hash.alg", limit=32) + content = _sl_strict_text(item.get("content"), "reference.hash.content", limit=256) + assert algorithm is not None and content is not None + if not re.fullmatch(r"[0-9a-f]{32,128}", content): + raise SBOMLineageError("reference.hash.content is not a digest") + checked_hashes.append({"alg": algorithm, "content": content}) + result["hashes"] = checked_hashes + return result + + +def _sl_validate_component_coverage(value: Any, *, has_version: bool, has_supplier: bool) -> dict[str, Any]: + _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "component.coverage") + state = value.get("state") + if not isinstance(state, str) or state not in _SL_COVERAGE: + raise SBOMLineageError("component.coverage.state is invalid") + unknown = _sl_string_list(value.get("unknown"), "component.coverage.unknown", maximum=32) + truncated = _sl_string_list(value.get("truncated"), "component.coverage.truncated", maximum=32) + expected_unknown = [] + if not has_version: + expected_unknown.append("version") + if not has_supplier: + expected_unknown.append("supplier") + if truncated: + expected_unknown.append("truncated:" + ",".join(truncated)) + expected_state = "complete" if not expected_unknown else "partial" + if state != expected_state or unknown != sorted(set(expected_unknown)): + raise SBOMLineageError("component coverage is inconsistent with its fields") + return {"state": state, "unknown": unknown, "truncated": truncated} + + +def _sl_validate_component(component: Any) -> dict[str, Any]: + required = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} + _sl_require_keys(component, required, set(), "component") + component_id_text = _sl_strict_text(component.get("component_id"), "component.component_id", limit=256) + assert component_id_text is not None + component_id = _sl_id(component_id_text, "component.component_id") + name = _sl_strict_text(component.get("name"), "component.name", limit=256) + version = _sl_strict_text(component.get("version"), "component.version", allow_none=True, limit=128) + supplier = _sl_strict_text(component.get("supplier"), "component.supplier", allow_none=True, limit=512) + identifiers = _sl_string_list(component.get("identifiers"), "component.identifiers", maximum=_SL_MAX_IDENTIFIERS) + if component_id not in identifiers: + raise SBOMLineageError("component.identifiers must bind component_id") + for identifier in identifiers: + _sl_strict_safe_locator(identifier, "component.identifier") + references = _sl_list(component.get("references"), "component.references") + if len(references) > _SL_MAX_REFERENCES: + raise SBOMLineageError("component.references exceeds its bound") + checked_references = [_sl_validate_reference(reference) for reference in references] + licenses = _sl_string_list(component.get("licenses"), "component.licenses", maximum=_SL_MAX_LICENSES) + component_type = _sl_strict_text(component.get("component_type"), "component.component_type", limit=64) + assert component_type is not None + coverage = _sl_validate_component_coverage( + component.get("coverage"), has_version=version is not None, has_supplier=supplier is not None, + ) + checked = { + "component_id": component_id, + "name": name, + "version": version, + "supplier": supplier, + "identifiers": identifiers, + "references": checked_references, + "licenses": licenses, + "component_type": component_type, + "coverage": coverage, + } + if _sl_json(checked) != _sl_json(dict(component)): + raise SBOMLineageError("component projection is not normalized") + return checked + + +def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") -> dict[str, Any]: + _sl_require_keys(relationship, {"from", "to", "type", "confidence", "coverage"}, {"evidence_refs"}, field) + source = _sl_strict_text(relationship.get("from"), f"{field}.from", limit=256) + target = _sl_strict_text(relationship.get("to"), f"{field}.to", limit=256) + rel_type = _sl_strict_text(relationship.get("type"), f"{field}.type", limit=96) + confidence = relationship.get("confidence") + coverage = relationship.get("coverage") + assert source is not None and target is not None and rel_type is not None + _sl_id(source, f"{field}.from") + _sl_id(target, f"{field}.to") + if rel_type != rel_type.casefold().replace(" ", "_"): + raise SBOMLineageError(f"{field}.type is not normalized") + if confidence not in _SL_CONFIDENCE or coverage not in _SL_COVERAGE: + raise SBOMLineageError(f"{field} confidence or coverage is invalid") + result: dict[str, Any] = { + "from": source, "to": target, "type": rel_type, + "confidence": confidence, "coverage": coverage, + } + if "evidence_refs" in relationship: + evidence_refs = _sl_string_list(relationship.get("evidence_refs"), f"{field}.evidence_refs", maximum=_SL_MAX_EVIDENCE_REFS) + if not evidence_refs: + raise SBOMLineageError(f"{field}.evidence_refs cannot be empty") + for ref in evidence_refs: + _sl_id(ref, f"{field}.evidence_ref") + result["evidence_refs"] = evidence_refs + if _sl_json(result) != _sl_json(dict(relationship)): + raise SBOMLineageError(f"{field} is not normalized") + return result + + +def _sl_expected_document_coverage(document_id: str, fmt: str, components: list[Mapping[str, Any]], relationships: list[Mapping[str, Any]], truncated: list[str]) -> dict[str, Any]: + component_ids = {item["component_id"] for item in components} + known_ids = set(component_ids) + if fmt == "SPDX": + known_ids.add(document_id) + dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) + unknown: list[str] = [] + if not components: + unknown.append("components") + if not relationships: + unknown.append("relationships") + if dangling: + unknown.append("dangling_relationships") + if truncated: + unknown.append("truncated:" + ",".join(sorted(set(truncated)))) + if any(item["coverage"]["state"] != "complete" for item in components): + unknown.append("component_metadata") + state = "unknown" if not components else "partial" if (dangling or truncated or not relationships or any(item["coverage"]["state"] != "complete" for item in components)) else "complete" + return { + "state": state, + "unknown": sorted(set(unknown)), + "component_count": len(components), + "relationship_count": len(relationships), + "truncated": sorted(set(truncated)), + "dangling_relationships": dangling, + } + + +def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: + required = { + "schema_version", "format", "spec_version", "document_id", "document_name", + "document_sha256", "source_ref", "created_at", "supplier", "components", + "relationships", "coverage", "ingestion_digest", + } + if not isinstance(document, Mapping) or set(document) != required or document.get("schema_version") != _SL_SCHEMA: + raise SBOMLineageError("normalized SBOM document shape is invalid") + supplied = document.get("ingestion_digest") + unsigned = dict(document) + unsigned.pop("ingestion_digest", None) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + raise SBOMLineageError("normalized SBOM ingestion digest mismatch") + fmt = document.get("format") + if fmt not in _SL_FORMATS: + raise SBOMLineageError("normalized SBOM format is invalid") + spec_version = document.get("spec_version") + if fmt == "SPDX": + _sl_validate_spdx_version(f"SPDX-{spec_version}") + else: + _sl_validate_cdx_version(spec_version) + document_id_text = _sl_strict_text(document.get("document_id"), "document_id", limit=256) + assert document_id_text is not None + document_id = _sl_id(document_id_text, "document_id") + for field, limit in (("document_name", 256), ("created_at", 128), ("supplier", 512)): + _sl_strict_text(document.get(field), field, allow_none=True, limit=limit) + document_sha = document.get("document_sha256") + if not isinstance(document_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", document_sha): + raise SBOMLineageError("normalized SBOM document digest is invalid") + _sl_strict_source_ref(document.get("source_ref")) + components = _sl_list(document.get("components"), "components") + relationships = _sl_list(document.get("relationships"), "relationships") + if len(components) > _SL_MAX_COMPONENTS or len(relationships) > _SL_MAX_RELATIONSHIPS: + raise SBOMLineageError("normalized SBOM collection exceeds its bound") + checked_components = [] + seen: set[str] = set() + for component in components: + checked = _sl_validate_component(component) + if checked["component_id"] in seen: + raise SBOMLineageError("normalized SBOM contains duplicate component IDs") + seen.add(checked["component_id"]) + checked_components.append(checked) + checked_relationships = [_sl_validate_relationship(item) for item in relationships] + coverage = document.get("coverage") + _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") + if coverage.get("state") not in _SL_COVERAGE: + raise SBOMLineageError("document.coverage.state is invalid") + _sl_string_list(coverage.get("unknown"), "document.coverage.unknown", maximum=64) + truncated = _sl_string_list(coverage.get("truncated"), "document.coverage.truncated", maximum=64) + dangling = _sl_string_list(coverage.get("dangling_relationships"), "document.coverage.dangling_relationships", maximum=_SL_MAX_RELATIONSHIPS) + _sl_nonnegative_int(coverage.get("component_count"), "document.coverage.component_count", maximum=_SL_MAX_COMPONENTS) + _sl_nonnegative_int(coverage.get("relationship_count"), "document.coverage.relationship_count", maximum=_SL_MAX_RELATIONSHIPS) + expected = _sl_expected_document_coverage(document_id, fmt, checked_components, checked_relationships, truncated) + if coverage != expected: + raise SBOMLineageError("document coverage is inconsistent with its contents") + return dict(document) + + +def verify_sbom_document(document: Mapping[str, Any]) -> dict[str, Any]: + try: + checked = _sl_validate_document(document) + except (SBOMLineageError, TypeError, ValueError) as exc: + return {"valid": False, "error": str(exc)} + return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} + + +def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: + if edges is None: + return [] + if not isinstance(edges, (list, tuple)): + raise SBOMLineageError("lineage edges must be a list") + result = [] + if len(edges) > _SL_MAX_EDGES: + _sl_truncated(truncated, "edges") + for raw in edges[:_SL_MAX_EDGES]: + if not isinstance(raw, Mapping): + raise SBOMLineageError("lineage edges must contain objects") + result.append(_sl_relationship( + raw.get("from", raw.get("source")), raw.get("to", raw.get("target")), raw.get("type", "related"), + confidence=raw.get("confidence", "unknown"), coverage=raw.get("coverage", "unknown"), + evidence_refs=raw.get("evidence_refs", []), truncated=truncated, + )) + return sorted(result, key=lambda item: (item["from"], item["to"], item["type"], item["confidence"], item["coverage"])) + + +def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: + """Build a deterministic graph from normalized or raw SBOM documents.""" + if not isinstance(documents, (list, tuple)) or not documents: + raise SBOMLineageError("at least one SBOM document is required") + truncated: list[str] = [] + if len(documents) > _SL_MAX_DOCUMENTS: + truncated.append("documents") + normalized = [] + for document in documents[:_SL_MAX_DOCUMENTS]: + if isinstance(document, Mapping) and document.get("schema_version") == _SL_SCHEMA: + normalized.append(_sl_validate_document(document)) + else: + normalized.append(ingest_sbom_document(document)) + nodes: dict[str, dict[str, Any]] = {} + component_fingerprints: dict[str, str] = {} + native_edges: list[dict[str, Any]] = [] + for document in normalized: + for component in document.get("components", []): + component_id = component["component_id"] + fingerprint = _sl_sha(component) + if component_id in component_fingerprints and component_fingerprints[component_id] != fingerprint: + raise SBOMLineageError(f"conflicting duplicate component ID: {component_id}") + if component_id in component_fingerprints: + continue + component_fingerprints[component_id] = fingerprint + node = dict(component) + node["node_id"] = component_id + node["kind"] = "component" + node["document_sha256"] = document["document_sha256"] + nodes[component_id] = node + native_edges.extend(document.get("relationships", [])) + for marker in document.get("coverage", {}).get("truncated", []): + _sl_truncated(truncated, marker) + external_edges = [] if edges is None else list(edges) if isinstance(edges, (list, tuple)) else None + if external_edges is None: + raise SBOMLineageError("lineage edges must be a list") + all_edges = _sl_lineage_edges(native_edges + external_edges, truncated=truncated) + for edge in all_edges: + for node_id in (edge["from"], edge["to"]): + nodes.setdefault(node_id, {"node_id": node_id, "kind": _sl_node_kind(node_id), "coverage": {"state": "partial", "unknown": ["node_metadata"], "truncated": []}}) + coverage_states = {edge["coverage"] for edge in all_edges} + document_states = {document.get("coverage", {}).get("state", "unknown") for document in normalized} + node_states = {node.get("coverage", {}).get("state", "complete") for node in nodes.values() if node.get("kind") == "component"} + if "unknown" in coverage_states or "unknown" in document_states or "unknown" in node_states: + coverage_state = "unknown" + elif truncated or "partial" in coverage_states or "partial" in document_states or "partial" in node_states: + coverage_state = "partial" + else: + coverage_state = "complete" + body: dict[str, Any] = { + "schema_version": _SL_LINEAGE_SCHEMA, + "documents": sorted(normalized, key=lambda item: item["document_sha256"]), + "nodes": [nodes[key] for key in sorted(nodes)], + "edges": all_edges, + "coverage": {"state": coverage_state, "unknown": [] if coverage_state == "complete" else ["external_lineage"], "truncated": sorted(set(truncated))}, + } + body["lineage_digest"] = _sl_sha(body) + return body + + +def _sl_validate_node_coverage(value: Any, *, component: bool) -> dict[str, Any]: + _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "node.coverage") + state = value.get("state") + if state not in _SL_COVERAGE: + raise SBOMLineageError("node.coverage.state is invalid") + unknown = _sl_string_list(value.get("unknown"), "node.coverage.unknown", maximum=32) + truncated = _sl_string_list(value.get("truncated"), "node.coverage.truncated", maximum=32) + if not component and (state != "partial" or unknown != ["node_metadata"] or truncated): + raise SBOMLineageError("external lineage node coverage is invalid") + return {"state": state, "unknown": unknown, "truncated": truncated} + + +def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[str, Mapping[str, Any]]]]) -> str: + if not isinstance(node, Mapping): + raise SBOMLineageError("lineage node must be an object") + node_id = _sl_strict_text(node.get("node_id"), "node.node_id", limit=256) + kind = _sl_strict_text(node.get("kind"), "node.kind", limit=32) + assert node_id is not None and kind is not None + _sl_id(node_id, "node.node_id") + expected_kind = _sl_node_kind(node_id) + if kind != expected_kind: + raise SBOMLineageError("lineage node kind does not match its ID") + component_fields = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} + if kind == "component" and component_fields.issubset(set(node)): + required = component_fields | {"node_id", "kind", "document_sha256"} + _sl_require_keys(node, required, set(), "component node") + component = {key: node[key] for key in component_fields} + checked = _sl_validate_component(component) + if checked["component_id"] != node_id: + raise SBOMLineageError("component node_id is not bound to component_id") + document_sha = node.get("document_sha256") + if not isinstance(document_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", document_sha): + raise SBOMLineageError("component node document digest is invalid") + candidates = component_map.get(node_id, []) + if not any(document_sha == digest and _sl_json(checked) == _sl_json(dict(source)) for digest, source in candidates): + raise SBOMLineageError("component node is not bound to a source document") + return node_id + if kind == "component": + _sl_require_keys(node, {"node_id", "kind", "coverage"}, set(), "component stub") + _sl_validate_node_coverage(node.get("coverage"), component=False) + if node_id in component_map: + raise SBOMLineageError("document-backed component node is incomplete") + return node_id + _sl_require_keys(node, {"node_id", "kind", "coverage"}, set(), "external node") + _sl_validate_node_coverage(node.get("coverage"), component=False) + return node_id + + +def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: + required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} + if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: + raise SBOMLineageError("unsupported software-lineage schema") + supplied = lineage.get("lineage_digest") + unsigned = dict(lineage) + unsigned.pop("lineage_digest", None) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + raise SBOMLineageError("lineage digest mismatch") + documents = _sl_list(lineage.get("documents"), "lineage.documents") + nodes = _sl_list(lineage.get("nodes"), "lineage.nodes") + edges = _sl_list(lineage.get("edges"), "lineage.edges") + if len(documents) > _SL_MAX_DOCUMENTS or len(edges) > _SL_MAX_EDGES or len(nodes) > _SL_MAX_NODES: + raise SBOMLineageError("lineage collection exceeds its bound") + checked_documents = [_sl_validate_document(document) for document in documents] + component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} + for document in checked_documents: + for component in document["components"]: + component_map.setdefault(component["component_id"], []).append((document["document_sha256"], component)) + for component_id, candidates in component_map.items(): + if len({_sl_sha(item) for _, item in candidates}) > 1: + raise SBOMLineageError(f"conflicting component projection for {component_id}") + node_ids: set[str] = set() + full_component_ids: set[str] = set() + for node in nodes: + node_id = _sl_validate_lineage_node(node, component_map) + if node_id in node_ids: + raise SBOMLineageError("lineage contains duplicate nodes") + node_ids.add(node_id) + if isinstance(node, Mapping) and "component_id" in node: + full_component_ids.add(node_id) + if full_component_ids != set(component_map): + raise SBOMLineageError("lineage component nodes are not bound to all documents") + checked_edges = [] + for edge in edges: + checked = _sl_validate_relationship(edge, field="lineage.edge") + if checked["from"] not in node_ids or checked["to"] not in node_ids: + raise SBOMLineageError("lineage edge endpoint is not bound to a node") + checked_edges.append(checked) + coverage = lineage.get("coverage") + _sl_require_keys(coverage, {"state", "unknown", "truncated"}, set(), "lineage.coverage") + state = coverage.get("state") + if state not in _SL_COVERAGE: + raise SBOMLineageError("lineage.coverage.state is invalid") + unknown = _sl_string_list(coverage.get("unknown"), "lineage.coverage.unknown", maximum=32) + truncated = _sl_string_list(coverage.get("truncated"), "lineage.coverage.truncated", maximum=64) + nested_truncated = {item for document in checked_documents for item in document["coverage"]["truncated"]} + if not nested_truncated.issubset(set(truncated)): + raise SBOMLineageError("lineage coverage lost document truncation metadata") + edge_states = {edge["coverage"] for edge in checked_edges} + document_states = {document["coverage"]["state"] for document in checked_documents} + node_states = {node["coverage"]["state"] for node in nodes if node.get("kind") == "component"} + expected_state = "unknown" if "unknown" in edge_states or "unknown" in document_states or "unknown" in node_states else "partial" if truncated or "partial" in edge_states or "partial" in document_states or "partial" in node_states else "complete" + expected_unknown = [] if expected_state == "complete" else ["external_lineage"] + if state != expected_state or unknown != expected_unknown: + raise SBOMLineageError("lineage coverage is inconsistent with its contents") + return dict(lineage) + + +def verify_sbom_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: + try: + loaded = _sl_loaded_lineage(lineage) + except (SBOMLineageError, TypeError, ValueError) as exc: + return {"valid": False, "error": str(exc)} + return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} + + +def _sl_query_matches(nodes: list[Mapping[str, Any]], query: str) -> list[str]: + term = _sl_strict_text(query, "query", limit=512) + assert term is not None + term = term.casefold() + matches = [] + for node in nodes: + values = [node.get("node_id", ""), node.get("name", ""), node.get("version", "") or ""] + values.extend(node.get("identifiers", []) if isinstance(node.get("identifiers"), list) else []) + references = node.get("references", []) + values.extend(ref.get("locator", "") for ref in references if isinstance(ref, Mapping)) + if any(term in str(value).casefold() for value in values): + matches.append(str(node["node_id"])) + return sorted(set(matches)) + + +def _sl_path_state(path: list[Mapping[str, Any]]) -> str: + states = {str(edge.get("coverage", "unknown")) for edge in path} + if "unknown" in states: + return "unknown" + if "partial" in states: + return "partial" + return "complete" + + +def _sl_path_confidence(path: list[Mapping[str, Any]]) -> str: + states = {str(edge.get("confidence", "unknown")) for edge in path} + if "unknown" in states: + return "unknown" + if "low" in states: + return "low" + if "medium" in states: + return "medium" + return "high" + + +def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32) -> dict[str, Any]: + """Find impacted artifact nodes and return every traversed evidence edge.""" + loaded = _sl_loaded_lineage(lineage) + limit = _sl_limit(limit, "limit") + query_text = _sl_strict_text(query, "query", limit=512) + assert query_text is not None + nodes = loaded.get("nodes", []) + edges = loaded.get("edges", []) + if not isinstance(nodes, list) or not isinstance(edges, list): + raise SBOMLineageError("lineage nodes and edges must be lists") + matches = _sl_query_matches(nodes, query_text) + truncated: list[str] = [] + if len(matches) > _SL_MAX_QUERY_MATCHES: + truncated.append("matched_nodes") + matches = matches[:_SL_MAX_QUERY_MATCHES] + adjacency: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} + for edge in edges: + if not isinstance(edge, Mapping): + continue + source, target = str(edge.get("from")), str(edge.get("to")) + adjacency.setdefault(source, []).append((target, edge)) + adjacency.setdefault(target, []).append((source, edge)) + found: dict[str, dict[str, Any]] = {} + path_truncated = False + for start in matches: + queue = deque([(start, [], {start})]) + while queue: + current, path, visited = queue.popleft() + if current != start and _sl_node_kind(current) == "artifact": + candidate = {"artifact_id": current, "path": path, "coverage": _sl_path_state(path), "confidence": _sl_path_confidence(path), "evidence_refs": sorted({ref for edge in path for ref in edge.get("evidence_refs", [])})} + previous = found.get(current) + if previous is None or len(candidate["path"]) < len(previous["path"]): + found[current] = candidate + continue + if len(path) >= _SL_MAX_PATH_LENGTH: + if adjacency.get(current): + path_truncated = True + continue + for neighbor, edge in sorted(adjacency.get(current, []), key=lambda item: (item[0], item[1].get("type", ""))): + if neighbor in visited: + continue + queue.append((neighbor, path + [dict(edge)], visited | {neighbor})) + all_impacted = [found[key] for key in sorted(found)] + if len(all_impacted) > limit: + truncated.append("impacted_artifacts") + impacted = all_impacted[:limit] + if path_truncated: + truncated.append("path") + global_state = loaded.get("coverage", {}).get("state", "unknown") + if truncated and global_state == "complete": + global_state = "partial" + path_states = {item["coverage"] for item in impacted} + if impacted: + status = "unknown" if "unknown" in path_states or global_state == "unknown" else "partial" if "partial" in path_states or global_state == "partial" else "complete" + if truncated or "unknown" in path_states: + impact_status = "not_established" + elif "partial" in path_states: + impact_status = "partial" + else: + impact_status = "established" + else: + status = "unknown" if global_state != "complete" else "not_found" + impact_status = "not_established" + unsigned: dict[str, Any] = { + "schema_version": _SL_QUERY_SCHEMA, + "query": query_text, + "matched_nodes": matches, + "impacted_artifacts": impacted, + "status": status, + "coverage": {"state": global_state, "path_states": sorted(path_states), "unknown": [] if global_state == "complete" and not truncated else ["lineage_completeness"], "truncated": sorted(set(truncated))}, + "claims": {"impact_status": impact_status, "not_affected": False, "negative_result_is_not_evidence": True}, + } + unsigned["query_digest"] = _sl_sha(unsigned) + return unsigned + + +def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any]: + required = {"schema_version", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} + if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: + raise SBOMLineageError("unsupported query schema") + supplied = query_result.get("query_digest") + unsigned = dict(query_result) + unsigned.pop("query_digest", None) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + raise SBOMLineageError("query digest mismatch") + query = _sl_strict_text(query_result.get("query"), "query", limit=512) + assert query is not None + matched_nodes = _sl_string_list(query_result.get("matched_nodes"), "matched_nodes", maximum=_SL_MAX_QUERY_MATCHES) + for node_id in matched_nodes: + _sl_id(node_id, "matched_node") + impacted = _sl_list(query_result.get("impacted_artifacts"), "impacted_artifacts") + if len(impacted) > _SL_MAX_QUERY_RESULTS: + raise SBOMLineageError("impacted_artifacts exceeds its bound") + impacted_ids: list[str] = [] + path_states: list[str] = [] + for item in impacted: + _sl_require_keys(item, {"artifact_id", "path", "coverage", "confidence", "evidence_refs"}, set(), "impacted_artifact") + artifact_id = _sl_strict_text(item.get("artifact_id"), "artifact_id", limit=256) + assert artifact_id is not None + _sl_id(artifact_id, "artifact_id") + if _sl_node_kind(artifact_id) != "artifact": + raise SBOMLineageError("impacted artifact ID is not an artifact") + if impacted_ids and artifact_id <= impacted_ids[-1]: + raise SBOMLineageError("impacted_artifacts must be unique and sorted") + impacted_ids.append(artifact_id) + path = _sl_list(item.get("path"), "impacted_artifact.path") + if not path or len(path) > _SL_MAX_PATH_LENGTH: + raise SBOMLineageError("impacted_artifact.path is outside its bounds") + checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] + if not any(set((edge["from"], edge["to"])) & set(matched_nodes) for edge in checked_path[:1]): + raise SBOMLineageError("query path is not bound to a matched node") + for first, second in zip(checked_path, checked_path[1:]): + if not set((first["from"], first["to"])) & set((second["from"], second["to"])): + raise SBOMLineageError("query path edges are not endpoint-bound") + if artifact_id not in {checked_path[-1]["from"], checked_path[-1]["to"]}: + raise SBOMLineageError("query path does not terminate at its artifact") + coverage = item.get("coverage") + confidence = item.get("confidence") + if coverage not in _SL_COVERAGE or confidence not in _SL_CONFIDENCE: + raise SBOMLineageError("query path result confidence or coverage is invalid") + expected_path_state = _sl_path_state(checked_path) + expected_path_confidence = _sl_path_confidence(checked_path) + if coverage != expected_path_state or confidence != expected_path_confidence: + raise SBOMLineageError("query path summary is inconsistent") + evidence_refs = _sl_string_list(item.get("evidence_refs"), "impacted_artifact.evidence_refs", maximum=_SL_MAX_EVIDENCE_REFS) + expected_refs = sorted({ref for edge in checked_path for ref in edge.get("evidence_refs", [])}) + if evidence_refs != expected_refs: + raise SBOMLineageError("query evidence references are not bound to its path") + path_states.append(coverage) + coverage = query_result.get("coverage") + _sl_require_keys(coverage, {"state", "path_states", "unknown", "truncated"}, set(), "query.coverage") + coverage_state = coverage.get("state") + if coverage_state not in _SL_COVERAGE: + raise SBOMLineageError("query.coverage.state is invalid") + declared_path_states = _sl_string_list(coverage.get("path_states"), "query.coverage.path_states", maximum=3) + if declared_path_states != sorted(set(path_states)): + raise SBOMLineageError("query coverage path states are inconsistent") + _sl_string_list(coverage.get("unknown"), "query.coverage.unknown", maximum=8) + truncated = _sl_string_list(coverage.get("truncated"), "query.coverage.truncated", maximum=16) + if truncated and coverage_state == "complete": + raise SBOMLineageError("query coverage cannot be complete when truncated") + expected_unknown = [] if coverage_state == "complete" and not truncated else ["lineage_completeness"] + if coverage.get("unknown") != expected_unknown: + raise SBOMLineageError("query coverage unknown state is inconsistent") + status = query_result.get("status") + if status not in {"complete", "partial", "unknown", "not_found"}: + raise SBOMLineageError("query status is invalid") + if impacted: + expected_status = "unknown" if "unknown" in path_states or coverage_state == "unknown" else "partial" if "partial" in path_states or coverage_state == "partial" else "complete" + expected_impact = "not_established" if truncated or "unknown" in path_states else "partial" if "partial" in path_states else "established" + else: + expected_status = "unknown" if coverage_state != "complete" else "not_found" + expected_impact = "not_established" + claims = query_result.get("claims") + _sl_require_keys(claims, {"impact_status", "not_affected", "negative_result_is_not_evidence"}, set(), "query.claims") + if claims.get("impact_status") != expected_impact or claims.get("not_affected") is not False or claims.get("negative_result_is_not_evidence") is not True: + raise SBOMLineageError("query claims are inconsistent with coverage") + if status != expected_status: + raise SBOMLineageError("query status is inconsistent with coverage") + return dict(query_result) + + +def verify_sbom_lineage_query(query_result: Mapping[str, Any]) -> dict[str, Any]: + try: + checked = _sl_validate_query_result(query_result) + except (SBOMLineageError, TypeError, ValueError) as exc: + return {"valid": False, "error": str(exc)} + return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} + + +def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: + """CLI adapter for local SBOM normalization, merge, and query operations.""" + try: + command = getattr(args, "sbom_command", None) + output = getattr(args, "output", None) + if command == "ingest": + document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) + elif command == "merge": + documents = [] + for path in args.documents: + raw_bytes = _sl_read_bounded(Path(path)) + try: + payload = json.loads(raw_bytes.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + payload = None + documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) + edges = json.loads(_sl_read_bounded(Path(args.edges)).decode("utf-8")) if getattr(args, "edges", None) else [] + document = build_sbom_lineage(documents, edges=edges) + elif command == "query": + lineage = json.loads(_sl_read_bounded(Path(args.lineage)).decode("utf-8")) + document = query_sbom_lineage(lineage, args.component, limit=getattr(args, "limit", 32)) + else: + raise SBOMLineageError("command must be ingest, merge, or query") + serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" + if output: + Path(output).write_text(serialized, encoding="utf-8") + if getattr(args, "json", False) or not output: + print(serialized, end="") + else: + digest_key = "lineage_digest" if "lineage_digest" in document else "query_digest" if "query_digest" in document else "ingestion_digest" + print(f"sbom {command} -> {output}\n{digest_key}: {document[digest_key]}") + return 0 + except (OSError, TypeError, ValueError, SBOMLineageError) as exc: + print(f"sbom: {exc}") + return 1 diff --git a/tests/fixtures/sbom/cyclonedx-app.json b/tests/fixtures/sbom/cyclonedx-app.json new file mode 100644 index 00000000..46811848 --- /dev/null +++ b/tests/fixtures/sbom/cyclonedx-app.json @@ -0,0 +1,40 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "serialNumber": "urn:uuid:perseus-build-json-001", + "version": 1, + "metadata": { + "timestamp": "2026-08-19T12:00:00Z", + "authors": [{"name": "Perseus Computing LLC"}], + "component": { + "type": "application", + "bom-ref": "pkg:generic/perseus-service@1.0.26", + "name": "perseus-service", + "version": "1.0.26", + "supplier": {"name": "Perseus Computing LLC"}, + "purl": "pkg:generic/perseus-service@1.0.26" + } + }, + "components": [ + { + "type": "library", + "bom-ref": "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0", + "group": "org.apache.logging.log4j", + "name": "log4j-core", + "version": "2.17.0", + "supplier": {"name": "Apache Software Foundation"}, + "purl": "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0", + "licenses": [{"license": {"id": "Apache-2.0"}}], + "externalReferences": [ + {"type": "advisories", "url": "https://nvd.nist.gov/vuln/detail/CVE-2021-44228"}, + {"type": "vulnerability", "url": "cve:CVE-2021-44228"} + ] + } + ], + "dependencies": [ + { + "ref": "pkg:generic/perseus-service@1.0.26", + "dependsOn": ["pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0"] + } + ] +} diff --git a/tests/fixtures/sbom/cyclonedx-app.xml b/tests/fixtures/sbom/cyclonedx-app.xml new file mode 100644 index 00000000..554de056 --- /dev/null +++ b/tests/fixtures/sbom/cyclonedx-app.xml @@ -0,0 +1,26 @@ + + + + 2026-08-19T12:00:00Z + Perseus Computing LLC + + perseus-service1.0.26 + Perseus Computing LLC + pkg:generic/perseus-service@1.0.26 + + + + + org.apache.logging.log4jlog4j-core2.17.0 + Apache Software Foundation + pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0 + Apache-2.0 + cve:CVE-2021-44228 + + + + + + + + diff --git a/tests/fixtures/sbom/spdx-app.json b/tests/fixtures/sbom/spdx-app.json new file mode 100644 index 00000000..6a3471ce --- /dev/null +++ b/tests/fixtures/sbom/spdx-app.json @@ -0,0 +1,57 @@ +{ + "spdxVersion": "SPDX-2.3", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "perseus-build", + "documentNamespace": "https://perseus.example/sbom/perseus-build-001", + "creationInfo": { + "created": "2026-08-19T12:00:00Z", + "creators": ["Organization: Perseus Computing LLC"] + }, + "packages": [ + { + "SPDXID": "SPDXRef-Log4j", + "name": "log4j-core", + "versionInfo": "2.17.0", + "supplier": "Organization: Apache Software Foundation", + "licenseConcluded": "Apache-2.0", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0" + }, + { + "referenceCategory": "SECURITY", + "referenceType": "cve", + "referenceLocator": "CVE-2021-44228" + } + ] + }, + { + "SPDXID": "SPDXRef-App", + "name": "perseus-service", + "versionInfo": "1.0.26", + "supplier": "Organization: Perseus Computing LLC", + "licenseConcluded": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/perseus-service@1.0.26" + } + ] + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-App" + }, + { + "spdxElementId": "SPDXRef-App", + "relationshipType": "DEPENDS_ON", + "relatedSpdxElement": "SPDXRef-Log4j" + } + ] +} diff --git a/tests/fixtures/sbom/spdx-app.xml b/tests/fixtures/sbom/spdx-app.xml new file mode 100644 index 00000000..5c1bba1a --- /dev/null +++ b/tests/fixtures/sbom/spdx-app.xml @@ -0,0 +1,28 @@ + + + SPDX-2.3 + SPDXRef-DOCUMENT + perseus-build-xml + https://perseus.example/sbom/perseus-build-xml-001 + + 2026-08-19T12:00:00Z + Organization: Perseus Computing LLC + + + SPDXRef-Log4j + log4j-core + 2.17.0 + Organization: Apache Software Foundation + Apache-2.0 + + PACKAGE-MANAGER + purl + pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0 + + + + SPDXRef-App + DEPENDS_ON + SPDXRef-Log4j + + diff --git a/tests/fixtures/sbom/spdx-rdf.xml b/tests/fixtures/sbom/spdx-rdf.xml new file mode 100644 index 00000000..a24e054c --- /dev/null +++ b/tests/fixtures/sbom/spdx-rdf.xml @@ -0,0 +1,39 @@ + + + + SPDX-2.3 + SPDXRef-DOCUMENT + perseus-rdf-build + + 2026-08-19T12:00:00Z + Organization: Perseus Computing LLC + + + SPDXRef-App + perseus-service + 1.0.26 + Organization: Perseus Computing LLC + + + SPDXRef-Log4j + log4j-core + 2.17.0 + Organization: Apache Software Foundation + + PACKAGE-MANAGER + purl + pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0 + + + + + DESCRIBES + + + + + DEPENDS_ON + + + + diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py new file mode 100644 index 00000000..07013cf3 --- /dev/null +++ b/tests/test_sbom_lineage.py @@ -0,0 +1,353 @@ +"""Offline SPDX/CycloneDX ingestion and software-lineage contract tests (#995).""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from conftest import perseus + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" / "sbom" + + +def _load(name: str) -> bytes: + return (FIXTURES / name).read_bytes() + + +def _lineage_edges() -> list[dict[str, object]]: + return [ + {"from": "source:perseus-repo", "to": "SPDXRef-Log4j", "type": "contains", "confidence": "high", "coverage": "complete"}, + {"from": "SPDXRef-Log4j", "to": "SPDXRef-App", "type": "used_by", "confidence": "high", "coverage": "complete"}, + {"from": "SPDXRef-App", "to": "build:perseus-001", "type": "built_into", "confidence": "medium", "coverage": "complete"}, + {"from": "build:perseus-001", "to": "artifact:perseus-image@1.0.26", "type": "generates", "confidence": "high", "coverage": "complete", "evidence_refs": ["ledger:receipt-001"]}, + {"from": "artifact:perseus-image@1.0.26", "to": "deployment:edge-001", "type": "deployed_as", "confidence": "unknown", "coverage": "partial"}, + ] + + +def test_ingests_spdx_json_with_digest_metadata_and_security_references(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture-spdx-json") + assert document["schema_version"] == "perseus-sbom/v1" + assert document["format"] == "SPDX" + assert document["spec_version"] == "2.3" + assert len(document["document_sha256"]) == 64 + log4j = next(item for item in document["components"] if item["name"] == "log4j-core") + assert log4j["version"] == "2.17.0" + assert "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0" in log4j["identifiers"] + assert any("CVE-2021-44228" in ref["locator"] for ref in log4j["references"]) + assert document["coverage"]["state"] == "complete" + + +def test_ingests_spdx_xml_and_cyclonedx_json_xml(): + spdx_xml = perseus.ingest_sbom_document(_load("spdx-app.xml"), source_ref="artifact:fixture-spdx-xml") + spdx_rdf = perseus.ingest_sbom_document(_load("spdx-rdf.xml"), source_ref="artifact:fixture-spdx-rdf") + cdx_json = perseus.ingest_sbom_document(_load("cyclonedx-app.json"), source_ref="artifact:fixture-cdx-json") + cdx_xml = perseus.ingest_sbom_document(_load("cyclonedx-app.xml"), source_ref="artifact:fixture-cdx-xml") + assert (spdx_xml["format"], spdx_xml["spec_version"]) == ("SPDX", "2.3") + assert (spdx_rdf["format"], spdx_rdf["spec_version"]) == ("SPDX", "2.3") + assert len(spdx_rdf["components"]) == 2 + assert (cdx_json["format"], cdx_json["spec_version"]) == ("CycloneDX", "1.5") + assert (cdx_xml["format"], cdx_xml["spec_version"]) == ("CycloneDX", "1.5") + assert all(any(item["name"] == "log4j-core" for item in doc["components"]) for doc in (spdx_xml, cdx_json, cdx_xml)) + + +def test_rejects_unknown_format_and_unsupported_version(): + with pytest.raises(perseus.SBOMLineageError, match="format"): + perseus.ingest_sbom_document(json.dumps({"format": "unknown", "version": "1"})) + with pytest.raises(perseus.SBOMLineageError, match="version"): + perseus.ingest_sbom_document(json.dumps({"spdxVersion": "SPDX-9.9", "SPDXID": "SPDXRef-DOCUMENT"})) + + +def test_query_returns_auditable_impacted_artifact_path_without_false_clean_claim(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture-spdx-json") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + assert result["status"] == "partial" + assert result["impacted_artifacts"][0]["artifact_id"] == "artifact:perseus-image@1.0.26" + assert len(result["impacted_artifacts"][0]["path"]) == 3 + assert [edge["type"] for edge in result["impacted_artifacts"][0]["path"]] == ["depends_on", "built_into", "generates"] + assert result["impacted_artifacts"][0]["coverage"] == "complete" + assert result["coverage"]["state"] == "partial" + assert result["claims"]["not_affected"] is False + assert result["claims"]["impact_status"] == "established" + verification = perseus.verify_sbom_lineage_query(result) + assert verification["valid"] is True + assert verification["expected_digest"] == result["query_digest"] + + +def test_incomplete_lineage_is_explicit_and_queryable(): + document = perseus.ingest_sbom_document(_load("cyclonedx-app.json"), source_ref="artifact:fixture-cdx-json") + lineage = perseus.build_sbom_lineage([document], edges=[{"from": "SPDXRef-Log4j", "to": "SPDXRef-App", "type": "used_by", "confidence": "unknown", "coverage": "unknown"}]) + result = perseus.query_sbom_lineage(lineage, "log4j-core") + assert result["status"] == "unknown" + assert result["impacted_artifacts"] == [] + assert result["claims"]["not_affected"] is False + assert result["claims"]["impact_status"] == "not_established" + assert result["coverage"]["state"] == "unknown" + + +def test_lineage_digest_is_deterministic_and_tamper_evident(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture-spdx-json") + first = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + second = perseus.build_sbom_lineage([document], edges=list(reversed(_lineage_edges()))) + assert first["lineage_digest"] == second["lineage_digest"] + tampered = json.loads(json.dumps(first)) + tampered["edges"][0]["coverage"] = "unknown" + assert perseus.verify_sbom_lineage(tampered)["valid"] is False + + +def test_cli_ingest_and_query_are_local_and_machine_readable(tmp_path): + document_path = tmp_path / "spdx.json" + document_path.write_bytes(_load("spdx-app.json")) + normalized_path = tmp_path / "normalized.json" + args = type("Args", (), {"sbom_command": "ingest", "document": str(document_path), "source_ref": "artifact:cli-fixture", "output": str(normalized_path), "json": False})() + assert perseus.cmd_sbom(args, {}) == 0 + assert json.loads(normalized_path.read_text())["format"] == "SPDX" + + +def test_credential_bearing_source_and_references_are_not_persisted(): + payload = json.loads(_load("spdx-app.json")) + payload["packages"][0]["externalRefs"].append({ + "referenceCategory": "SECURITY", + "referenceType": "website", + "referenceLocator": "https://user:pw@example.invalid/?token=RAWSECRET", + }) + document = perseus.ingest_sbom_document(payload, source_ref="Authorization: Bearer RAWSECRET") + encoded = json.dumps(document, sort_keys=True) + assert "RAWSECRET" not in encoded + assert "Authorization" not in encoded + assert any(ref["locator"].startswith("sha256:") for component in document["components"] for ref in component["references"]) + assert document["source_ref"].startswith("sha256:") + + +def test_untrusted_normalized_documents_and_conflicting_duplicate_ids_fail_closed(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + tampered = json.loads(json.dumps(document)) + tampered["components"][0]["name"] = "forged" + with pytest.raises(perseus.SBOMLineageError, match="digest"): + perseus.build_sbom_lineage([tampered]) + alternate = json.loads(_load("spdx-app.json")) + alternate["packages"][0]["name"] = "different-component" + alternate_document = perseus.ingest_sbom_document(alternate, source_ref="artifact:alternate") + with pytest.raises(perseus.SBOMLineageError, match="duplicate component ID"): + perseus.build_sbom_lineage([document, alternate_document]) + + +def test_input_size_is_bounded_before_parsing(): + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus.ingest_sbom_document(b"{" + b" " * (8 * 1024 * 1024) + b"}") + + +def test_truncation_and_dangling_relationships_downgrade_coverage(): + packages = [{"SPDXID": f"SPDXRef-Pkg{i}", "name": f"pkg-{i}", "versionInfo": "1.0", "supplier": "Organization: Test"} for i in range(513)] + payload = {"spdxVersion": "SPDX-2.3", "SPDXID": "SPDXRef-DOCUMENT", "packages": packages, "relationships": [{"spdxElementId": "SPDXRef-DOCUMENT", "relationshipType": "DESCRIBES", "relatedSpdxElement": "SPDXRef-Pkg0"}]} + document = perseus.ingest_sbom_document(payload, source_ref="artifact:truncated") + assert document["coverage"]["state"] == "partial" + assert "packages" in document["coverage"]["truncated"] + payload["packages"] = payload["packages"][:1] + payload["relationships"][0]["relatedSpdxElement"] = "SPDXRef-Missing" + dangling = perseus.ingest_sbom_document(payload, source_ref="artifact:dangling") + assert dangling["coverage"]["state"] == "partial" + assert dangling["coverage"]["dangling_relationships"] + assert perseus.build_sbom_lineage([dangling])["coverage"]["state"] == "partial" + + +def test_cyclonedx_reference_without_url_is_rejected(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [{"type": "vulnerability", "urlType": "cve"}] + with pytest.raises(perseus.SBOMLineageError, match="requires url"): + perseus.ingest_sbom_document(payload) + + +def _reseal(mapping): + unsigned = dict(mapping) + unsigned.pop("ingestion_digest", None) + unsigned.pop("lineage_digest", None) + unsigned.pop("query_digest", None) + if "schema_version" in mapping and mapping["schema_version"] == "perseus-sbom/v1": + mapping["ingestion_digest"] = perseus._sl_sha(unsigned) + elif "schema_version" in mapping and mapping["schema_version"] == "perseus-software-lineage/v1": + mapping["lineage_digest"] = perseus._sl_sha(unsigned) + else: + mapping["query_digest"] = perseus._sl_sha(unsigned) + return mapping + + +def test_all_untrusted_reference_surfaces_are_sanitized_in_json_and_xml(): + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["purl"] = "pkg:maven/example/pkg@1?api_key=RAW_PURL" + component["externalReferences"] = [{ + "type": "other", + "url": "https://user:pw@example.invalid/pkg?query=RAW_URL", + "comment": "Bearer RAW_COMMENT", + "hashes": [{"alg": "SHA-256", "content": "basic RAW_HASH"}], + }] + component["properties"] = [{"name": "vex", "value": "https://source.invalid/?query=RAW_PROPERTY"}] + document = perseus.ingest_sbom_document(payload, source_ref="https://source.invalid/?query=RAW_SOURCE") + encoded = json.dumps(document, sort_keys=True) + for secret in ("RAW_PURL", "RAW_URL", "RAW_COMMENT", "RAW_HASH", "RAW_PROPERTY", "RAW_SOURCE"): + assert secret not in encoded + assert any(item["locator"].startswith("sha256:") for item in document["components"][0]["references"]) + + spdx_xml = _load("spdx-app.xml").decode("utf-8").replace( + "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0", + "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0?api_key=RAW_XML_PURL", + ) + xml_document = perseus.ingest_sbom_document(spdx_xml) + assert "RAW_XML_PURL" not in json.dumps(xml_document, sort_keys=True) + + +def test_digest_valid_normalized_document_with_unsafe_source_or_fields_fails_closed(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + tampered = json.loads(json.dumps(document)) + tampered["source_ref"] = "https://source.invalid/?api_key=RAW_SOURCE" + tampered["components"][0]["identifiers"].append("pkg:generic/foo@1?token=RAW_IDENTIFIER") + tampered["components"][0]["references"][0]["comment"] = "basic RAW_COMMENT" + tampered["components"][0]["references"][0]["hashes"] = [{"alg": "SHA-256", "content": "Bearer RAW_HASH"}] + _reseal(tampered) + with pytest.raises(perseus.SBOMLineageError, match="unsafe|sanit"): + perseus.build_sbom_lineage([tampered]) + + +def test_path_size_is_checked_before_reading_path_input(tmp_path, monkeypatch): + oversized = tmp_path / "oversized.json" + with oversized.open("wb") as handle: + handle.truncate(perseus._SL_MAX_INPUT_BYTES + 1) + + def unexpected_read(*args, **kwargs): + raise AssertionError("oversized input was opened before its size was checked") + + monkeypatch.setattr(Path, "read_bytes", unexpected_read) + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus.ingest_sbom_document(oversized) + + +@pytest.mark.parametrize("command", ["ingest", "merge", "query"]) +def test_cli_checks_file_sizes_before_reading_ingest_merge_and_query(tmp_path, monkeypatch, command): + oversized = tmp_path / f"{command}.json" + with oversized.open("wb") as handle: + handle.truncate(perseus._SL_MAX_INPUT_BYTES + 1) + + def unexpected_read(*args, **kwargs): + raise AssertionError("CLI opened oversized input before checking its size") + + monkeypatch.setattr(Path, "read_bytes", unexpected_read) + monkeypatch.setattr(Path, "read_text", unexpected_read) + if command == "ingest": + args = type("Args", (), {"sbom_command": "ingest", "document": str(oversized), "source_ref": "", "output": None, "json": True})() + elif command == "merge": + args = type("Args", (), {"sbom_command": "merge", "documents": [str(oversized)], "edges": None, "output": None, "json": True})() + else: + args = type("Args", (), {"sbom_command": "query", "lineage": str(oversized), "component": "x", "limit": 32, "output": None, "json": True})() + assert perseus.cmd_sbom(args, {}) == 1 + + +def test_malformed_collections_missing_ids_and_duplicate_ids_are_rejected(): + payload = json.loads(_load("spdx-app.json")) + payload["packages"] = {"not": "a list"} + with pytest.raises(perseus.SBOMLineageError, match="packages"): + perseus.ingest_sbom_document(payload) + + payload = json.loads(_load("spdx-app.json")) + del payload["packages"][0]["SPDXID"] + with pytest.raises(perseus.SBOMLineageError, match="ID"): + perseus.ingest_sbom_document(payload) + + payload = json.loads(_load("spdx-app.json")) + payload["packages"].append(json.loads(json.dumps(payload["packages"][0]))) + with pytest.raises(perseus.SBOMLineageError, match="duplicate"): + perseus.ingest_sbom_document(payload) + + payload = json.loads(_load("spdx-app.json")) + del payload["SPDXID"] + with pytest.raises(perseus.SBOMLineageError, match="ID"): + perseus.ingest_sbom_document(payload) + + +def test_cyclonedx_xml_does_not_overwrite_duplicate_or_fallback_component_ids(): + xml = _load("cyclonedx-app.xml").decode("utf-8") + duplicate = 'log4j-core2.17.0' + with pytest.raises(perseus.SBOMLineageError, match="duplicate"): + perseus.ingest_sbom_document(xml.replace("", duplicate + "")) + + missing = xml.replace(' bom-ref="pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0"', "").replace( + "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0", "", + ) + with pytest.raises(perseus.SBOMLineageError, match="ID"): + perseus.ingest_sbom_document(missing) + + +def test_all_collection_caps_are_recorded_and_downgrade_coverage(): + payload = json.loads(_load("spdx-app.json")) + payload["packages"][0]["externalRefs"] = [ + {"referenceCategory": "SECURITY", "referenceType": "website", "referenceLocator": f"https://example.invalid/{i}"} + for i in range(65) + ] + document = perseus.ingest_sbom_document(payload) + assert "externalRefs" in document["coverage"]["truncated"] + assert document["coverage"]["state"] == "partial" + + lineage = perseus.build_sbom_lineage([document] * 65, edges=[]) + assert "documents" in lineage["coverage"]["truncated"] + assert lineage["coverage"]["state"] != "complete" + + edges = [{"from": f"source:s{i}", "to": f"artifact:a{i}", "type": "generates", "confidence": "high", "coverage": "complete"} for i in range(4097)] + edge_lineage = perseus.build_sbom_lineage([document], edges=edges) + assert "edges" in edge_lineage["coverage"]["truncated"] + assert edge_lineage["coverage"]["state"] != "complete" + + +def test_query_result_cap_is_recorded_and_cannot_claim_established_impact(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + edges = _lineage_edges() + [ + {"from": "SPDXRef-App", "to": "build:perseus-002", "type": "built_into", "confidence": "high", "coverage": "complete"}, + {"from": "build:perseus-002", "to": "artifact:perseus-image@2.0.0", "type": "generates", "confidence": "high", "coverage": "complete"}, + ] + lineage = perseus.build_sbom_lineage([document], edges=edges) + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228", limit=1) + assert result["coverage"]["truncated"] + assert result["status"] != "complete" + assert result["claims"]["impact_status"] != "established" + + +def test_document_lineage_and_query_verifiers_check_consistency_and_bindings(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + invalid_document = json.loads(json.dumps(document)) + invalid_document["coverage"]["component_count"] = 999 + _reseal(invalid_document) + assert perseus.verify_sbom_document(invalid_document)["valid"] is False + + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + invalid_lineage = json.loads(json.dumps(lineage)) + artifact = next(node["node_id"] for node in invalid_lineage["nodes"] if node["node_id"].startswith("artifact:")) + invalid_lineage["nodes"] = [node for node in invalid_lineage["nodes"] if node["node_id"] != artifact] + _reseal(invalid_lineage) + assert perseus.verify_sbom_lineage(invalid_lineage)["valid"] is False + + invalid_coverage = json.loads(json.dumps(lineage)) + invalid_coverage["coverage"] = {"state": "complete", "unknown": [], "truncated": []} + _reseal(invalid_coverage) + assert perseus.verify_sbom_lineage(invalid_coverage)["valid"] is False + + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + invalid_query = json.loads(json.dumps(result)) + invalid_query["impacted_artifacts"][0]["path"][0]["from"] = "source:unbound" + _reseal(invalid_query) + assert perseus.verify_sbom_lineage_query(invalid_query)["valid"] is False + + +def test_query_numeric_bounds_and_complete_schema_are_fail_closed(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + with pytest.raises(perseus.SBOMLineageError, match="limit"): + perseus.query_sbom_lineage(lineage, "log4j-core", limit=0) + with pytest.raises(perseus.SBOMLineageError, match="limit"): + perseus.query_sbom_lineage(lineage, "log4j-core", limit=257) + + invalid = json.loads(json.dumps(document)) + del invalid["components"][0]["coverage"]["unknown"] + _reseal(invalid) + assert perseus.verify_sbom_document(invalid)["valid"] is False From e9f9ac2d4f1fa7329da771e18cb832ba3e34501a Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 16:03:50 +0000 Subject: [PATCH 02/40] fix(sbom): harden lineage ingestion and verification (#995) --- perseus.py | 319 ++++++++++++++++++++++++++++++------ src/perseus/sbom_lineage.py | 317 +++++++++++++++++++++++++++++------ tests/test_sbom_lineage.py | 130 +++++++++++++++ 3 files changed, 665 insertions(+), 101 deletions(-) diff --git a/perseus.py b/perseus.py index 9167021e..5e8934ed 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "da2025e-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "a599b56-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44461,6 +44461,7 @@ def cmd_code_map(args, cfg) -> int: _SL_CONFIDENCE = frozenset({"high", "medium", "low", "unknown"}) _SL_COVERAGE = frozenset({"complete", "partial", "unknown"}) _SL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") +_SL_PURL_ID_RE = re.compile(r"^pkg:[A-Za-z0-9][A-Za-z0-9.+-]{0,31}/[^\s]{1,240}$") _SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") @@ -44488,6 +44489,28 @@ def cmd_code_map(args, cfg) -> int: _SL_MAX_QUERY_RESULTS = 256 _SL_MAX_PATH_LENGTH = 32 _SL_MAX_NODES = 20_000 +_SL_MAX_JSON_DEPTH = 128 +_SL_MAX_QUERY_STATES = 16_384 +_SL_MAX_QUERY_QUEUE = 4_096 +_SL_HASH_ALGORITHM_ALIASES = { + "md5": "md5", + "sha1": "sha-1", + "sha-1": "sha-1", + "sha224": "sha-224", + "sha-224": "sha-224", + "sha256": "sha-256", + "sha-256": "sha-256", + "sha384": "sha-384", + "sha-384": "sha-384", + "sha512": "sha-512", + "sha-512": "sha-512", + "sha3-256": "sha3-256", + "sha3-384": "sha3-384", + "sha3-512": "sha3-512", + "blake2b-256": "blake2b-256", + "blake2s-256": "blake2s-256", + "blake3": "blake3", +} class SBOMLineageError(ValueError): @@ -44502,10 +44525,19 @@ def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() +def _sl_ingestion_sha(unsigned: Mapping[str, Any]) -> str: + return _sl_sha({"domain": "perseus-sbom-ingestion", "document_sha256": unsigned.get("document_sha256"), "projection": unsigned}) + + def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" authority = value.split("://", 1)[1].split("/", 1)[0] if "://" in value else "" - return bool(_SL_SENSITIVE_REFERENCE_RE.search(value) or (authority and "@" in authority)) + normalized = re.sub(r"[^a-z0-9]+", "_", value.casefold()) + return bool( + _SL_SENSITIVE_REFERENCE_RE.search(value) + or (authority and "@" in authority) + or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) + ) def _sl_truncated(truncated: list[str] | None, name: str) -> None: @@ -44553,7 +44585,13 @@ def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512 def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: text = _sl_text(value if value is not None else fallback, field, required=True, limit=256) - if not _SL_ID_RE.fullmatch(text): + if _sl_sensitive(text): + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + prefix = text.split(":", 1)[0] + if prefix in {"source", "artifact", "deployment", "build", "vault", "ledger", "file"}: + return f"{prefix}:sha256:{digest}" + return f"sha256:{digest}" + if not _SL_ID_RE.fullmatch(text) and not _SL_PURL_ID_RE.fullmatch(text): raise SBOMLineageError(f"{field} is not a bounded identifier") return text @@ -44570,11 +44608,23 @@ def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: if value in (None, ""): return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" text = _sl_text(value, "source_ref", required=True, limit=512) - if text.startswith("sha256:") and re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text): + raw_digest = hashlib.sha256(raw_bytes).hexdigest() + if text.startswith("sha256:"): + if not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text) or text[7:].casefold() != raw_digest: + raise SBOMLineageError("source_ref digest is not bound to the SBOM bytes") return text.lower() - if _SL_PUBLIC_SOURCE_RE.fullmatch(text) and not _SL_SENSITIVE_REFERENCE_RE.search(text): + if _SL_PUBLIC_SOURCE_RE.fullmatch(text) and not _sl_sensitive(text): return text - return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + return "sha256:source-ref:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sl_hash_algorithm(value: Any, field: str = "hash_algorithm", *, strict: bool = False) -> str | None: + text = _sl_text(value, field, required=True, limit=32) + normalized = re.sub(r"[_ ]+", "-", text.casefold()) + result = _SL_HASH_ALGORITHM_ALIASES.get(normalized) + if result is None and strict: + raise SBOMLineageError(f"{field} is unsupported") + return result def _sl_strict_text(value: Any, field: str, *, allow_none: bool = False, limit: int = 512) -> str | None: @@ -44603,7 +44653,7 @@ def _sl_strict_source_ref(value: Any) -> str: text = _sl_strict_text(value, "source_ref", limit=512) assert text is not None if text.startswith("sha256:"): - if not re.fullmatch(r"sha256:[0-9a-f]{64}", text): + if not re.fullmatch(r"sha256:[0-9a-f]{64}", text) and not re.fullmatch(r"sha256:source-ref:[0-9a-f]{64}", text): raise SBOMLineageError("source_ref digest is malformed") return text if not _SL_PUBLIC_SOURCE_RE.fullmatch(text) or _sl_sensitive(text): @@ -44621,12 +44671,14 @@ def _sl_safe_hashes(value: Any, *, truncated: list[str] | None = None) -> list[d for raw in hashes[:_SL_MAX_HASHES]: if not isinstance(raw, Mapping): raise SBOMLineageError("hashes must contain objects") - algorithm = _sl_text(raw.get("alg"), "hash_algorithm", required=True, limit=32) + algorithm = _sl_hash_algorithm(raw.get("alg")) content = _sl_text(raw.get("content"), "hash", required=True, limit=256) # Only retain actual digest-looking values. Arbitrary hash content is # an attacker-controlled data channel and is deliberately dropped. - if re.fullmatch(r"[0-9a-fA-F]{32,128}", content): + if algorithm is not None and re.fullmatch(r"[0-9a-fA-F]{32,128}", content): result.append({"alg": algorithm, "content": content.lower()}) + else: + _sl_truncated(truncated, "hashes") return result @@ -44686,11 +44738,18 @@ def _sl_descendants(root: Any, *names: str) -> list[Any]: def _sl_supplier(value: Any) -> str: if isinstance(value, Mapping): - return _sl_text(value.get("name"), "supplier") + return _sl_safe_text(value.get("name"), "supplier") if isinstance(value, list): names = [_sl_supplier(item) for item in value] return "; ".join(item for item in names if item) - return _sl_text(value, "supplier") + return _sl_safe_text(value, "supplier") + + +def _sl_safe_text(value: Any, field: str, *, required: bool = False, limit: int = 512) -> str: + text = _sl_text(value, field, required=required, limit=limit) + if _sl_sensitive(text): + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + return text def _sl_normalize_reference(reference_type: Any, locator: Any, *, category: Any = "", comment: Any = None, hashes: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: @@ -44700,7 +44759,7 @@ def _sl_normalize_reference(reference_type: Any, locator: Any, *, category: Any "type": ref_type if ref_type in _SL_REFERENCE_TYPES else "other", "locator": locator_text, } - category_text = _sl_text(category, "reference_category", limit=64) + category_text = _sl_safe_text(category, "reference_category", limit=64) if category_text: result["category"] = category_text safe_hashes = _sl_safe_hashes(hashes, truncated=truncated) @@ -44771,11 +44830,17 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di raise SBOMLineageError("properties must contain objects") name = _sl_text(raw.get("name"), "property_name", limit=128) prop_value = _sl_text(raw.get("value"), "property_value", required=True, limit=1024) - if name and prop_value and any(marker in name.casefold() for marker in ("vex", "vuln", "signature", "attestation")): + normalized_name = re.sub(r"[^a-z0-9]+", "_", name.casefold()) + category = next((marker for marker in ("vex", "vulnerability", "vuln", "signature", "attestation") if marker in normalized_name), "") + if category == "vuln": + category = "vulnerability" + if category and prop_value: result.append({ - "type": name, + "type": category, "locator": "sha256:" + hashlib.sha256(prop_value.encode("utf-8")).hexdigest(), }) + elif name or prop_value: + _sl_truncated(truncated, "properties") return result @@ -44791,8 +44856,8 @@ def _sl_component( licenses: Any = None, truncated: list[str] | None = None, ) -> dict[str, Any]: - normalized_name = _sl_text(name, "component_name", required=True, limit=256) - normalized_version = _sl_text(version, "component_version", limit=128) + normalized_name = _sl_safe_text(name, "component_name", required=True, limit=256) + normalized_version = _sl_safe_text(version, "component_version", limit=128) normalized_id = _sl_id(component_id, "component ID") ids = {normalized_id} component_truncated: list[str] = list(truncated or []) @@ -44830,11 +44895,11 @@ def _sl_component( item = item.get("license", item.get("id")) if isinstance(item, Mapping): item = item.get("id") or item.get("name") - text = _sl_text(item, "license", limit=128) + text = _sl_safe_text(item, "license", limit=128) if text: license_values.append(text) elif licenses: - text = _sl_text(licenses, "license", limit=128) + text = _sl_safe_text(licenses, "license", limit=128) if text: license_values.append(text) unknown = [] @@ -44861,7 +44926,7 @@ def _sl_component( def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") - rel_type = _sl_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") + rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") if not isinstance(confidence, str) or confidence not in _SL_CONFIDENCE: raise SBOMLineageError("relationship confidence is unsupported") if not isinstance(coverage, str) or coverage not in _SL_COVERAGE: @@ -44942,6 +45007,10 @@ def _sl_validate_cdx_version(value: Any) -> str: def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") + if len(components) > _SL_MAX_COMPONENTS: + raise SBOMLineageError("normalized SBOM components exceed their global bound") + if len(relationships) > _SL_MAX_RELATIONSHIPS: + raise SBOMLineageError("normalized SBOM relationships exceed their global bound") component_ids: set[str] = set() for item in components: component_id = item.get("component_id") if isinstance(item, Mapping) else None @@ -44958,6 +45027,12 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("components") if not relationships: unknown.append("relationships") + if not document_name: + unknown.append("document_name") + if not created_at: + unknown.append("created_at") + if not supplier: + unknown.append("supplier") if dangling: unknown.append("dangling_relationships") if truncation: @@ -44966,7 +45041,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("component_metadata") if not components: coverage_state = "unknown" - elif dangling or truncation or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: + elif dangling or truncation or not document_name or not created_at or not supplier or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: coverage_state = "partial" else: coverage_state = "complete" @@ -44975,10 +45050,10 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "format": fmt, "spec_version": spec_version, "document_id": document_id, - "document_name": document_name or None, + "document_name": _sl_safe_text(document_name, "document_name", limit=256) or None, "document_sha256": hashlib.sha256(raw_bytes).hexdigest(), "source_ref": source_ref, - "created_at": created_at or None, + "created_at": _sl_safe_text(created_at, "created_at", limit=128) or None, "supplier": supplier or None, "components": sorted(components, key=lambda item: (item["component_id"] == metadata_component_id, item["component_id"])), "relationships": sorted(relationships, key=lambda item: (item["from"], item["to"], item["type"])), @@ -44991,7 +45066,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "dangling_relationships": dangling, }, } - unsigned["ingestion_digest"] = _sl_sha(unsigned) + unsigned["ingestion_digest"] = _sl_ingestion_sha(unsigned) return unsigned @@ -45053,11 +45128,12 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s raw_components.append(metadata_component) raw_component_list = _sl_list(value.get("components"), "components") raw_dependency_list = _sl_list(value.get("dependencies"), "dependencies") - if len(raw_component_list) > _SL_MAX_COMPONENTS: + component_budget = _SL_MAX_COMPONENTS - (1 if metadata_component else 0) + if len(raw_component_list) > component_budget: truncated.append("components") if len(raw_dependency_list) > _SL_MAX_RELATIONSHIPS: truncated.append("dependencies") - for item in raw_component_list[:_SL_MAX_COMPONENTS]: + for item in raw_component_list[:component_budget]: if not isinstance(item, Mapping): raise SBOMLineageError("components must contain objects") raw_components.append(item) @@ -45072,7 +45148,14 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s targets = _sl_list(raw.get("dependsOn"), "dependsOn") if len(targets) > _SL_MAX_DEPENDENCY_TARGETS: truncated.append("dependency_edges") - for target in targets[:_SL_MAX_DEPENDENCY_TARGETS]: + remaining = _SL_MAX_RELATIONSHIPS - len(relationships) + if remaining <= 0: + truncated.append("dependency_edges") + break + allowed_targets = min(_SL_MAX_DEPENDENCY_TARGETS, remaining) + if len(targets) > allowed_targets: + truncated.append("dependency_edges") + for target in targets[:allowed_targets]: relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated)) creators = metadata.get("authors", []) if not isinstance(creators, list): @@ -45081,7 +45164,7 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s document_id = _sl_id(value.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, - document_name="CycloneDX BOM", created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, + document_name=_sl_text(metadata.get("name"), "document_name"), created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, metadata_component_id=components[0]["component_id"] if metadata_component else "", ) @@ -45257,9 +45340,10 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, components_node = _sl_child(root, "components") if components_node is not None: component_nodes = _sl_children(components_node, "component") - if len(component_nodes) > _SL_MAX_COMPONENTS: + component_budget = _SL_MAX_COMPONENTS - (1 if metadata_component is not None else 0) + if len(component_nodes) > component_budget: truncated.append("components") - raw_components.extend(component_nodes[:_SL_MAX_COMPONENTS]) + raw_components.extend(component_nodes[:component_budget]) components = [] for raw in raw_components: components.append(_sl_cdx_xml_component(raw, truncated=truncated)) @@ -45274,7 +45358,14 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, children = _sl_children(dependency, "dependency") if len(children) > _SL_MAX_DEPENDENCY_TARGETS: truncated.append("dependency_edges") - for child in children[:_SL_MAX_DEPENDENCY_TARGETS]: + remaining = _SL_MAX_RELATIONSHIPS - len(relationships) + if remaining <= 0: + truncated.append("dependency_edges") + break + allowed_targets = min(_SL_MAX_DEPENDENCY_TARGETS, remaining) + if len(children) > allowed_targets: + truncated.append("dependency_edges") + for child in children[:allowed_targets]: relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated)) timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" authors = _sl_child(metadata, "authors") if metadata is not None else None @@ -45282,7 +45373,7 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, - document_name="CycloneDX BOM", created_at=timestamp, supplier=author, + document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, metadata_component_id=components[0]["component_id"] if metadata_component is not None else "", ) @@ -45305,6 +45396,61 @@ def _sl_read_bounded(path: Path) -> bytes: return raw_bytes +def _sl_validate_json_depth(raw_bytes: bytes) -> None: + depth = 0 + in_string = False + escaped = False + for byte in raw_bytes: + if in_string: + if escaped: + escaped = False + elif byte == 0x5C: + escaped = True + elif byte == 0x22: + in_string = False + continue + if byte == 0x22: + in_string = True + elif byte in (0x7B, 0x5B): + depth += 1 + if depth > _SL_MAX_JSON_DEPTH: + raise SBOMLineageError("SBOM JSON nesting is too deep") + elif byte in (0x7D, 0x5D): + depth = max(0, depth - 1) + + +def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: + if re.search(rb" _SL_MAX_XML_ELEMENTS: + raise SBOMLineageError("SBOM XML contains too many elements") + if depth > _SL_MAX_XML_DEPTH: + raise SBOMLineageError("SBOM XML nesting is too deep") + if root is None: + root = element + else: + depth -= 1 + parsed = parser.close() + except _sl_et.ParseError: + raise + if parsed is not None: + root = parsed + if root is None: + raise _sl_et.ParseError("empty XML document") + return root + + def _sl_payload(document: Any) -> tuple[Any, bytes]: if isinstance(document, Path): return _sl_payload(_sl_read_bounded(document)) @@ -45329,14 +45475,16 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: if not raw_bytes.strip(): raise SBOMLineageError("SBOM input is empty") try: + _sl_validate_json_depth(raw_bytes) value = json.loads(raw_bytes.decode("utf-8")) if not isinstance(value, Mapping): raise SBOMLineageError("SBOM JSON root must be an object") return dict(value), raw_bytes + except RecursionError as exc: + raise SBOMLineageError("SBOM JSON nesting is too deep") from exc except (UnicodeDecodeError, json.JSONDecodeError): try: - root = _sl_et.fromstring(raw_bytes) - _sl_validate_xml_tree(root) + root = _sl_parse_xml_bounded(raw_bytes) except _sl_et.ParseError as exc: raise SBOMLineageError("SBOM is neither valid JSON nor XML") from exc return root, raw_bytes @@ -45416,7 +45564,7 @@ def _sl_validate_reference(reference: Any) -> dict[str, Any]: checked_hashes = [] for item in hashes: _sl_require_keys(item, {"alg", "content"}, set(), "reference.hash") - algorithm = _sl_strict_text(item.get("alg"), "reference.hash.alg", limit=32) + algorithm = _sl_hash_algorithm(item.get("alg"), "reference.hash.alg", strict=True) content = _sl_strict_text(item.get("content"), "reference.hash.content", limit=256) assert algorithm is not None and content is not None if not re.fullmatch(r"[0-9a-f]{32,128}", content): @@ -45516,7 +45664,7 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") return result -def _sl_expected_document_coverage(document_id: str, fmt: str, components: list[Mapping[str, Any]], relationships: list[Mapping[str, Any]], truncated: list[str]) -> dict[str, Any]: +def _sl_expected_document_coverage(document_id: str, fmt: str, document_name: str | None, created_at: str | None, supplier: str | None, components: list[dict[str, Any]], relationships: list[dict[str, Any]], truncated: list[str]) -> dict[str, Any]: component_ids = {item["component_id"] for item in components} known_ids = set(component_ids) if fmt == "SPDX": @@ -45527,13 +45675,19 @@ def _sl_expected_document_coverage(document_id: str, fmt: str, components: list[ unknown.append("components") if not relationships: unknown.append("relationships") + if not document_name: + unknown.append("document_name") + if not created_at: + unknown.append("created_at") + if not supplier: + unknown.append("supplier") if dangling: unknown.append("dangling_relationships") if truncated: unknown.append("truncated:" + ",".join(sorted(set(truncated)))) if any(item["coverage"]["state"] != "complete" for item in components): unknown.append("component_metadata") - state = "unknown" if not components else "partial" if (dangling or truncated or not relationships or any(item["coverage"]["state"] != "complete" for item in components)) else "complete" + state = "unknown" if not components else "partial" if (dangling or truncated or not document_name or not created_at or not supplier or not relationships or any(item["coverage"]["state"] != "complete" for item in components)) else "complete" return { "state": state, "unknown": sorted(set(unknown)), @@ -45555,8 +45709,9 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: supplied = document.get("ingestion_digest") unsigned = dict(document) unsigned.pop("ingestion_digest", None) - if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or supplied not in {_sl_ingestion_sha(unsigned), _sl_sha(unsigned)}: raise SBOMLineageError("normalized SBOM ingestion digest mismatch") + legacy_ingestion_digest = supplied == _sl_sha(unsigned) fmt = document.get("format") if fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") @@ -45573,7 +45728,9 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: document_sha = document.get("document_sha256") if not isinstance(document_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", document_sha): raise SBOMLineageError("normalized SBOM document digest is invalid") - _sl_strict_source_ref(document.get("source_ref")) + source_ref = _sl_strict_source_ref(document.get("source_ref")) + if re.fullmatch(r"sha256:[0-9a-f]{64}", source_ref) and source_ref[7:] != document_sha: + raise SBOMLineageError("normalized source_ref is not bound to document bytes") components = _sl_list(document.get("components"), "components") relationships = _sl_list(document.get("relationships"), "relationships") if len(components) > _SL_MAX_COMPONENTS or len(relationships) > _SL_MAX_RELATIONSHIPS: @@ -45596,9 +45753,14 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: dangling = _sl_string_list(coverage.get("dangling_relationships"), "document.coverage.dangling_relationships", maximum=_SL_MAX_RELATIONSHIPS) _sl_nonnegative_int(coverage.get("component_count"), "document.coverage.component_count", maximum=_SL_MAX_COMPONENTS) _sl_nonnegative_int(coverage.get("relationship_count"), "document.coverage.relationship_count", maximum=_SL_MAX_RELATIONSHIPS) - expected = _sl_expected_document_coverage(document_id, fmt, checked_components, checked_relationships, truncated) + expected = _sl_expected_document_coverage( + document_id, fmt, document.get("document_name"), document.get("created_at"), document.get("supplier"), + checked_components, checked_relationships, truncated, + ) if coverage != expected: raise SBOMLineageError("document coverage is inconsistent with its contents") + if legacy_ingestion_digest: + raise SBOMLineageError("normalized SBOM ingestion digest is not bound to document bytes") return dict(document) @@ -45626,7 +45788,36 @@ def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list confidence=raw.get("confidence", "unknown"), coverage=raw.get("coverage", "unknown"), evidence_refs=raw.get("evidence_refs", []), truncated=truncated, )) - return sorted(result, key=lambda item: (item["from"], item["to"], item["type"], item["confidence"], item["coverage"])) + confidence_rank = {"high": 3, "medium": 2, "low": 1, "unknown": 0} + coverage_rank = {"complete": 2, "partial": 1, "unknown": 0} + selected: dict[tuple[str, str, str], dict[str, Any]] = {} + for edge in result: + key = (edge["from"], edge["to"], edge["type"]) + evidence = tuple(edge.get("evidence_refs", [])) + candidate_rank = ( + 0 if evidence else 1, + -len(evidence), + -confidence_rank[edge["confidence"]], + -coverage_rank[edge["coverage"]], + evidence, + _sl_sha(edge), + ) + current = selected.get(key) + if current is None: + selected[key] = edge + continue + current_evidence = tuple(current.get("evidence_refs", [])) + current_rank = ( + 0 if current_evidence else 1, + -len(current_evidence), + -confidence_rank[current["confidence"]], + -coverage_rank[current["coverage"]], + current_evidence, + _sl_sha(current), + ) + if candidate_rank < current_rank: + selected[key] = edge + return sorted(selected.values(), key=lambda item: (item["from"], item["to"], item["type"], _sl_sha(item))) def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: @@ -45671,7 +45862,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: nodes.setdefault(node_id, {"node_id": node_id, "kind": _sl_node_kind(node_id), "coverage": {"state": "partial", "unknown": ["node_metadata"], "truncated": []}}) coverage_states = {edge["coverage"] for edge in all_edges} document_states = {document.get("coverage", {}).get("state", "unknown") for document in normalized} - node_states = {node.get("coverage", {}).get("state", "complete") for node in nodes.values() if node.get("kind") == "component"} + node_states = {node.get("coverage", {}).get("state", "complete") for node in nodes.values()} if "unknown" in coverage_states or "unknown" in document_states or "unknown" in node_states: coverage_state = "unknown" elif truncated or "partial" in coverage_states or "partial" in document_states or "partial" in node_states: @@ -45788,7 +45979,7 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: raise SBOMLineageError("lineage coverage lost document truncation metadata") edge_states = {edge["coverage"] for edge in checked_edges} document_states = {document["coverage"]["state"] for document in checked_documents} - node_states = {node["coverage"]["state"] for node in nodes if node.get("kind") == "component"} + node_states = {node["coverage"]["state"] for node in nodes} expected_state = "unknown" if "unknown" in edge_states or "unknown" in document_states or "unknown" in node_states else "partial" if truncated or "partial" in edge_states or "partial" in document_states or "partial" in node_states else "complete" expected_unknown = [] if expected_state == "complete" else ["external_lineage"] if state != expected_state or unknown != expected_unknown: @@ -45860,13 +46051,30 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 continue source, target = str(edge.get("from")), str(edge.get("to")) adjacency.setdefault(source, []).append((target, edge)) - adjacency.setdefault(target, []).append((source, edge)) + reverse = dict(edge) + reverse["from"], reverse["to"] = target, source + adjacency.setdefault(target, []).append((source, reverse)) found: dict[str, dict[str, Any]] = {} path_truncated = False + states_used = 0 + queued_states = 0 + budget_exhausted = False for start in matches: - queue = deque([(start, [], {start})]) + if queued_states >= _SL_MAX_QUERY_QUEUE: + path_truncated = True + truncated.append("query_queue") + break + queue = deque([(start, [])]) + queued_states += 1 + seen_nodes = {start} while queue: - current, path, visited = queue.popleft() + if states_used >= _SL_MAX_QUERY_STATES: + path_truncated = True + truncated.append("query_states") + budget_exhausted = True + break + current, path = queue.popleft() + states_used += 1 if current != start and _sl_node_kind(current) == "artifact": candidate = {"artifact_id": current, "path": path, "coverage": _sl_path_state(path), "confidence": _sl_path_confidence(path), "evidence_refs": sorted({ref for edge in path for ref in edge.get("evidence_refs", [])})} previous = found.get(current) @@ -45878,9 +46086,18 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 path_truncated = True continue for neighbor, edge in sorted(adjacency.get(current, []), key=lambda item: (item[0], item[1].get("type", ""))): - if neighbor in visited: + if neighbor in seen_nodes: continue - queue.append((neighbor, path + [dict(edge)], visited | {neighbor})) + seen_nodes.add(neighbor) + if len(queue) >= _SL_MAX_QUERY_QUEUE or queued_states >= _SL_MAX_QUERY_QUEUE: + path_truncated = True + truncated.append("query_queue") + budget_exhausted = True + continue + queue.append((neighbor, path + [dict(edge)])) + queued_states += 1 + if budget_exhausted: + break all_impacted = [found[key] for key in sorted(found)] if len(all_impacted) > limit: truncated.append("impacted_artifacts") @@ -45948,12 +46165,12 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] if not path or len(path) > _SL_MAX_PATH_LENGTH: raise SBOMLineageError("impacted_artifact.path is outside its bounds") checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] - if not any(set((edge["from"], edge["to"])) & set(matched_nodes) for edge in checked_path[:1]): + if checked_path[0]["from"] not in set(matched_nodes): raise SBOMLineageError("query path is not bound to a matched node") for first, second in zip(checked_path, checked_path[1:]): - if not set((first["from"], first["to"])) & set((second["from"], second["to"])): - raise SBOMLineageError("query path edges are not endpoint-bound") - if artifact_id not in {checked_path[-1]["from"], checked_path[-1]["to"]}: + if first["to"] != second["from"]: + raise SBOMLineageError("query path edges are not directed and contiguous") + if checked_path[-1]["to"] != artifact_id: raise SBOMLineageError("query path does not terminate at its artifact") coverage = item.get("coverage") confidence = item.get("confidence") diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 2f02ec77..6b371df6 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -25,6 +25,7 @@ _SL_CONFIDENCE = frozenset({"high", "medium", "low", "unknown"}) _SL_COVERAGE = frozenset({"complete", "partial", "unknown"}) _SL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") +_SL_PURL_ID_RE = re.compile(r"^pkg:[A-Za-z0-9][A-Za-z0-9.+-]{0,31}/[^\s]{1,240}$") _SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") @@ -52,6 +53,28 @@ _SL_MAX_QUERY_RESULTS = 256 _SL_MAX_PATH_LENGTH = 32 _SL_MAX_NODES = 20_000 +_SL_MAX_JSON_DEPTH = 128 +_SL_MAX_QUERY_STATES = 16_384 +_SL_MAX_QUERY_QUEUE = 4_096 +_SL_HASH_ALGORITHM_ALIASES = { + "md5": "md5", + "sha1": "sha-1", + "sha-1": "sha-1", + "sha224": "sha-224", + "sha-224": "sha-224", + "sha256": "sha-256", + "sha-256": "sha-256", + "sha384": "sha-384", + "sha-384": "sha-384", + "sha512": "sha-512", + "sha-512": "sha-512", + "sha3-256": "sha3-256", + "sha3-384": "sha3-384", + "sha3-512": "sha3-512", + "blake2b-256": "blake2b-256", + "blake2s-256": "blake2s-256", + "blake3": "blake3", +} class SBOMLineageError(ValueError): @@ -66,10 +89,19 @@ def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() +def _sl_ingestion_sha(unsigned: Mapping[str, Any]) -> str: + return _sl_sha({"domain": "perseus-sbom-ingestion", "document_sha256": unsigned.get("document_sha256"), "projection": unsigned}) + + def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" authority = value.split("://", 1)[1].split("/", 1)[0] if "://" in value else "" - return bool(_SL_SENSITIVE_REFERENCE_RE.search(value) or (authority and "@" in authority)) + normalized = re.sub(r"[^a-z0-9]+", "_", value.casefold()) + return bool( + _SL_SENSITIVE_REFERENCE_RE.search(value) + or (authority and "@" in authority) + or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) + ) def _sl_truncated(truncated: list[str] | None, name: str) -> None: @@ -117,7 +149,13 @@ def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512 def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: text = _sl_text(value if value is not None else fallback, field, required=True, limit=256) - if not _SL_ID_RE.fullmatch(text): + if _sl_sensitive(text): + digest = hashlib.sha256(text.encode("utf-8")).hexdigest() + prefix = text.split(":", 1)[0] + if prefix in {"source", "artifact", "deployment", "build", "vault", "ledger", "file"}: + return f"{prefix}:sha256:{digest}" + return f"sha256:{digest}" + if not _SL_ID_RE.fullmatch(text) and not _SL_PURL_ID_RE.fullmatch(text): raise SBOMLineageError(f"{field} is not a bounded identifier") return text @@ -134,11 +172,23 @@ def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: if value in (None, ""): return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" text = _sl_text(value, "source_ref", required=True, limit=512) - if text.startswith("sha256:") and re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text): + raw_digest = hashlib.sha256(raw_bytes).hexdigest() + if text.startswith("sha256:"): + if not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text) or text[7:].casefold() != raw_digest: + raise SBOMLineageError("source_ref digest is not bound to the SBOM bytes") return text.lower() - if _SL_PUBLIC_SOURCE_RE.fullmatch(text) and not _SL_SENSITIVE_REFERENCE_RE.search(text): + if _SL_PUBLIC_SOURCE_RE.fullmatch(text) and not _sl_sensitive(text): return text - return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + return "sha256:source-ref:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _sl_hash_algorithm(value: Any, field: str = "hash_algorithm", *, strict: bool = False) -> str | None: + text = _sl_text(value, field, required=True, limit=32) + normalized = re.sub(r"[_ ]+", "-", text.casefold()) + result = _SL_HASH_ALGORITHM_ALIASES.get(normalized) + if result is None and strict: + raise SBOMLineageError(f"{field} is unsupported") + return result def _sl_strict_text(value: Any, field: str, *, allow_none: bool = False, limit: int = 512) -> str | None: @@ -167,7 +217,7 @@ def _sl_strict_source_ref(value: Any) -> str: text = _sl_strict_text(value, "source_ref", limit=512) assert text is not None if text.startswith("sha256:"): - if not re.fullmatch(r"sha256:[0-9a-f]{64}", text): + if not re.fullmatch(r"sha256:[0-9a-f]{64}", text) and not re.fullmatch(r"sha256:source-ref:[0-9a-f]{64}", text): raise SBOMLineageError("source_ref digest is malformed") return text if not _SL_PUBLIC_SOURCE_RE.fullmatch(text) or _sl_sensitive(text): @@ -185,12 +235,14 @@ def _sl_safe_hashes(value: Any, *, truncated: list[str] | None = None) -> list[d for raw in hashes[:_SL_MAX_HASHES]: if not isinstance(raw, Mapping): raise SBOMLineageError("hashes must contain objects") - algorithm = _sl_text(raw.get("alg"), "hash_algorithm", required=True, limit=32) + algorithm = _sl_hash_algorithm(raw.get("alg")) content = _sl_text(raw.get("content"), "hash", required=True, limit=256) # Only retain actual digest-looking values. Arbitrary hash content is # an attacker-controlled data channel and is deliberately dropped. - if re.fullmatch(r"[0-9a-fA-F]{32,128}", content): + if algorithm is not None and re.fullmatch(r"[0-9a-fA-F]{32,128}", content): result.append({"alg": algorithm, "content": content.lower()}) + else: + _sl_truncated(truncated, "hashes") return result @@ -250,11 +302,18 @@ def _sl_descendants(root: Any, *names: str) -> list[Any]: def _sl_supplier(value: Any) -> str: if isinstance(value, Mapping): - return _sl_text(value.get("name"), "supplier") + return _sl_safe_text(value.get("name"), "supplier") if isinstance(value, list): names = [_sl_supplier(item) for item in value] return "; ".join(item for item in names if item) - return _sl_text(value, "supplier") + return _sl_safe_text(value, "supplier") + + +def _sl_safe_text(value: Any, field: str, *, required: bool = False, limit: int = 512) -> str: + text = _sl_text(value, field, required=required, limit=limit) + if _sl_sensitive(text): + return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() + return text def _sl_normalize_reference(reference_type: Any, locator: Any, *, category: Any = "", comment: Any = None, hashes: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: @@ -264,7 +323,7 @@ def _sl_normalize_reference(reference_type: Any, locator: Any, *, category: Any "type": ref_type if ref_type in _SL_REFERENCE_TYPES else "other", "locator": locator_text, } - category_text = _sl_text(category, "reference_category", limit=64) + category_text = _sl_safe_text(category, "reference_category", limit=64) if category_text: result["category"] = category_text safe_hashes = _sl_safe_hashes(hashes, truncated=truncated) @@ -335,11 +394,17 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di raise SBOMLineageError("properties must contain objects") name = _sl_text(raw.get("name"), "property_name", limit=128) prop_value = _sl_text(raw.get("value"), "property_value", required=True, limit=1024) - if name and prop_value and any(marker in name.casefold() for marker in ("vex", "vuln", "signature", "attestation")): + normalized_name = re.sub(r"[^a-z0-9]+", "_", name.casefold()) + category = next((marker for marker in ("vex", "vulnerability", "vuln", "signature", "attestation") if marker in normalized_name), "") + if category == "vuln": + category = "vulnerability" + if category and prop_value: result.append({ - "type": name, + "type": category, "locator": "sha256:" + hashlib.sha256(prop_value.encode("utf-8")).hexdigest(), }) + elif name or prop_value: + _sl_truncated(truncated, "properties") return result @@ -355,8 +420,8 @@ def _sl_component( licenses: Any = None, truncated: list[str] | None = None, ) -> dict[str, Any]: - normalized_name = _sl_text(name, "component_name", required=True, limit=256) - normalized_version = _sl_text(version, "component_version", limit=128) + normalized_name = _sl_safe_text(name, "component_name", required=True, limit=256) + normalized_version = _sl_safe_text(version, "component_version", limit=128) normalized_id = _sl_id(component_id, "component ID") ids = {normalized_id} component_truncated: list[str] = list(truncated or []) @@ -394,11 +459,11 @@ def _sl_component( item = item.get("license", item.get("id")) if isinstance(item, Mapping): item = item.get("id") or item.get("name") - text = _sl_text(item, "license", limit=128) + text = _sl_safe_text(item, "license", limit=128) if text: license_values.append(text) elif licenses: - text = _sl_text(licenses, "license", limit=128) + text = _sl_safe_text(licenses, "license", limit=128) if text: license_values.append(text) unknown = [] @@ -425,7 +490,7 @@ def _sl_component( def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") - rel_type = _sl_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") + rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") if not isinstance(confidence, str) or confidence not in _SL_CONFIDENCE: raise SBOMLineageError("relationship confidence is unsupported") if not isinstance(coverage, str) or coverage not in _SL_COVERAGE: @@ -506,6 +571,10 @@ def _sl_validate_cdx_version(value: Any) -> str: def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") + if len(components) > _SL_MAX_COMPONENTS: + raise SBOMLineageError("normalized SBOM components exceed their global bound") + if len(relationships) > _SL_MAX_RELATIONSHIPS: + raise SBOMLineageError("normalized SBOM relationships exceed their global bound") component_ids: set[str] = set() for item in components: component_id = item.get("component_id") if isinstance(item, Mapping) else None @@ -522,6 +591,12 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("components") if not relationships: unknown.append("relationships") + if not document_name: + unknown.append("document_name") + if not created_at: + unknown.append("created_at") + if not supplier: + unknown.append("supplier") if dangling: unknown.append("dangling_relationships") if truncation: @@ -530,7 +605,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("component_metadata") if not components: coverage_state = "unknown" - elif dangling or truncation or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: + elif dangling or truncation or not document_name or not created_at or not supplier or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: coverage_state = "partial" else: coverage_state = "complete" @@ -539,10 +614,10 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "format": fmt, "spec_version": spec_version, "document_id": document_id, - "document_name": document_name or None, + "document_name": _sl_safe_text(document_name, "document_name", limit=256) or None, "document_sha256": hashlib.sha256(raw_bytes).hexdigest(), "source_ref": source_ref, - "created_at": created_at or None, + "created_at": _sl_safe_text(created_at, "created_at", limit=128) or None, "supplier": supplier or None, "components": sorted(components, key=lambda item: (item["component_id"] == metadata_component_id, item["component_id"])), "relationships": sorted(relationships, key=lambda item: (item["from"], item["to"], item["type"])), @@ -555,7 +630,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "dangling_relationships": dangling, }, } - unsigned["ingestion_digest"] = _sl_sha(unsigned) + unsigned["ingestion_digest"] = _sl_ingestion_sha(unsigned) return unsigned @@ -617,11 +692,12 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s raw_components.append(metadata_component) raw_component_list = _sl_list(value.get("components"), "components") raw_dependency_list = _sl_list(value.get("dependencies"), "dependencies") - if len(raw_component_list) > _SL_MAX_COMPONENTS: + component_budget = _SL_MAX_COMPONENTS - (1 if metadata_component else 0) + if len(raw_component_list) > component_budget: truncated.append("components") if len(raw_dependency_list) > _SL_MAX_RELATIONSHIPS: truncated.append("dependencies") - for item in raw_component_list[:_SL_MAX_COMPONENTS]: + for item in raw_component_list[:component_budget]: if not isinstance(item, Mapping): raise SBOMLineageError("components must contain objects") raw_components.append(item) @@ -636,7 +712,14 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s targets = _sl_list(raw.get("dependsOn"), "dependsOn") if len(targets) > _SL_MAX_DEPENDENCY_TARGETS: truncated.append("dependency_edges") - for target in targets[:_SL_MAX_DEPENDENCY_TARGETS]: + remaining = _SL_MAX_RELATIONSHIPS - len(relationships) + if remaining <= 0: + truncated.append("dependency_edges") + break + allowed_targets = min(_SL_MAX_DEPENDENCY_TARGETS, remaining) + if len(targets) > allowed_targets: + truncated.append("dependency_edges") + for target in targets[:allowed_targets]: relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated)) creators = metadata.get("authors", []) if not isinstance(creators, list): @@ -645,7 +728,7 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s document_id = _sl_id(value.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, - document_name="CycloneDX BOM", created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, + document_name=_sl_text(metadata.get("name"), "document_name"), created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, metadata_component_id=components[0]["component_id"] if metadata_component else "", ) @@ -821,9 +904,10 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, components_node = _sl_child(root, "components") if components_node is not None: component_nodes = _sl_children(components_node, "component") - if len(component_nodes) > _SL_MAX_COMPONENTS: + component_budget = _SL_MAX_COMPONENTS - (1 if metadata_component is not None else 0) + if len(component_nodes) > component_budget: truncated.append("components") - raw_components.extend(component_nodes[:_SL_MAX_COMPONENTS]) + raw_components.extend(component_nodes[:component_budget]) components = [] for raw in raw_components: components.append(_sl_cdx_xml_component(raw, truncated=truncated)) @@ -838,7 +922,14 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, children = _sl_children(dependency, "dependency") if len(children) > _SL_MAX_DEPENDENCY_TARGETS: truncated.append("dependency_edges") - for child in children[:_SL_MAX_DEPENDENCY_TARGETS]: + remaining = _SL_MAX_RELATIONSHIPS - len(relationships) + if remaining <= 0: + truncated.append("dependency_edges") + break + allowed_targets = min(_SL_MAX_DEPENDENCY_TARGETS, remaining) + if len(children) > allowed_targets: + truncated.append("dependency_edges") + for child in children[:allowed_targets]: relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated)) timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" authors = _sl_child(metadata, "authors") if metadata is not None else None @@ -846,7 +937,7 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, - document_name="CycloneDX BOM", created_at=timestamp, supplier=author, + document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, metadata_component_id=components[0]["component_id"] if metadata_component is not None else "", ) @@ -869,6 +960,61 @@ def _sl_read_bounded(path: Path) -> bytes: return raw_bytes +def _sl_validate_json_depth(raw_bytes: bytes) -> None: + depth = 0 + in_string = False + escaped = False + for byte in raw_bytes: + if in_string: + if escaped: + escaped = False + elif byte == 0x5C: + escaped = True + elif byte == 0x22: + in_string = False + continue + if byte == 0x22: + in_string = True + elif byte in (0x7B, 0x5B): + depth += 1 + if depth > _SL_MAX_JSON_DEPTH: + raise SBOMLineageError("SBOM JSON nesting is too deep") + elif byte in (0x7D, 0x5D): + depth = max(0, depth - 1) + + +def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: + if re.search(rb" _SL_MAX_XML_ELEMENTS: + raise SBOMLineageError("SBOM XML contains too many elements") + if depth > _SL_MAX_XML_DEPTH: + raise SBOMLineageError("SBOM XML nesting is too deep") + if root is None: + root = element + else: + depth -= 1 + parsed = parser.close() + except _sl_et.ParseError: + raise + if parsed is not None: + root = parsed + if root is None: + raise _sl_et.ParseError("empty XML document") + return root + + def _sl_payload(document: Any) -> tuple[Any, bytes]: if isinstance(document, Path): return _sl_payload(_sl_read_bounded(document)) @@ -893,14 +1039,16 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: if not raw_bytes.strip(): raise SBOMLineageError("SBOM input is empty") try: + _sl_validate_json_depth(raw_bytes) value = json.loads(raw_bytes.decode("utf-8")) if not isinstance(value, Mapping): raise SBOMLineageError("SBOM JSON root must be an object") return dict(value), raw_bytes + except RecursionError as exc: + raise SBOMLineageError("SBOM JSON nesting is too deep") from exc except (UnicodeDecodeError, json.JSONDecodeError): try: - root = _sl_et.fromstring(raw_bytes) - _sl_validate_xml_tree(root) + root = _sl_parse_xml_bounded(raw_bytes) except _sl_et.ParseError as exc: raise SBOMLineageError("SBOM is neither valid JSON nor XML") from exc return root, raw_bytes @@ -980,7 +1128,7 @@ def _sl_validate_reference(reference: Any) -> dict[str, Any]: checked_hashes = [] for item in hashes: _sl_require_keys(item, {"alg", "content"}, set(), "reference.hash") - algorithm = _sl_strict_text(item.get("alg"), "reference.hash.alg", limit=32) + algorithm = _sl_hash_algorithm(item.get("alg"), "reference.hash.alg", strict=True) content = _sl_strict_text(item.get("content"), "reference.hash.content", limit=256) assert algorithm is not None and content is not None if not re.fullmatch(r"[0-9a-f]{32,128}", content): @@ -1080,7 +1228,7 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") return result -def _sl_expected_document_coverage(document_id: str, fmt: str, components: list[Mapping[str, Any]], relationships: list[Mapping[str, Any]], truncated: list[str]) -> dict[str, Any]: +def _sl_expected_document_coverage(document_id: str, fmt: str, document_name: str | None, created_at: str | None, supplier: str | None, components: list[dict[str, Any]], relationships: list[dict[str, Any]], truncated: list[str]) -> dict[str, Any]: component_ids = {item["component_id"] for item in components} known_ids = set(component_ids) if fmt == "SPDX": @@ -1091,13 +1239,19 @@ def _sl_expected_document_coverage(document_id: str, fmt: str, components: list[ unknown.append("components") if not relationships: unknown.append("relationships") + if not document_name: + unknown.append("document_name") + if not created_at: + unknown.append("created_at") + if not supplier: + unknown.append("supplier") if dangling: unknown.append("dangling_relationships") if truncated: unknown.append("truncated:" + ",".join(sorted(set(truncated)))) if any(item["coverage"]["state"] != "complete" for item in components): unknown.append("component_metadata") - state = "unknown" if not components else "partial" if (dangling or truncated or not relationships or any(item["coverage"]["state"] != "complete" for item in components)) else "complete" + state = "unknown" if not components else "partial" if (dangling or truncated or not document_name or not created_at or not supplier or not relationships or any(item["coverage"]["state"] != "complete" for item in components)) else "complete" return { "state": state, "unknown": sorted(set(unknown)), @@ -1119,8 +1273,9 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: supplied = document.get("ingestion_digest") unsigned = dict(document) unsigned.pop("ingestion_digest", None) - if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or supplied not in {_sl_ingestion_sha(unsigned), _sl_sha(unsigned)}: raise SBOMLineageError("normalized SBOM ingestion digest mismatch") + legacy_ingestion_digest = supplied == _sl_sha(unsigned) fmt = document.get("format") if fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") @@ -1137,7 +1292,9 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: document_sha = document.get("document_sha256") if not isinstance(document_sha, str) or not re.fullmatch(r"[0-9a-f]{64}", document_sha): raise SBOMLineageError("normalized SBOM document digest is invalid") - _sl_strict_source_ref(document.get("source_ref")) + source_ref = _sl_strict_source_ref(document.get("source_ref")) + if re.fullmatch(r"sha256:[0-9a-f]{64}", source_ref) and source_ref[7:] != document_sha: + raise SBOMLineageError("normalized source_ref is not bound to document bytes") components = _sl_list(document.get("components"), "components") relationships = _sl_list(document.get("relationships"), "relationships") if len(components) > _SL_MAX_COMPONENTS or len(relationships) > _SL_MAX_RELATIONSHIPS: @@ -1160,9 +1317,14 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: dangling = _sl_string_list(coverage.get("dangling_relationships"), "document.coverage.dangling_relationships", maximum=_SL_MAX_RELATIONSHIPS) _sl_nonnegative_int(coverage.get("component_count"), "document.coverage.component_count", maximum=_SL_MAX_COMPONENTS) _sl_nonnegative_int(coverage.get("relationship_count"), "document.coverage.relationship_count", maximum=_SL_MAX_RELATIONSHIPS) - expected = _sl_expected_document_coverage(document_id, fmt, checked_components, checked_relationships, truncated) + expected = _sl_expected_document_coverage( + document_id, fmt, document.get("document_name"), document.get("created_at"), document.get("supplier"), + checked_components, checked_relationships, truncated, + ) if coverage != expected: raise SBOMLineageError("document coverage is inconsistent with its contents") + if legacy_ingestion_digest: + raise SBOMLineageError("normalized SBOM ingestion digest is not bound to document bytes") return dict(document) @@ -1190,7 +1352,36 @@ def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list confidence=raw.get("confidence", "unknown"), coverage=raw.get("coverage", "unknown"), evidence_refs=raw.get("evidence_refs", []), truncated=truncated, )) - return sorted(result, key=lambda item: (item["from"], item["to"], item["type"], item["confidence"], item["coverage"])) + confidence_rank = {"high": 3, "medium": 2, "low": 1, "unknown": 0} + coverage_rank = {"complete": 2, "partial": 1, "unknown": 0} + selected: dict[tuple[str, str, str], dict[str, Any]] = {} + for edge in result: + key = (edge["from"], edge["to"], edge["type"]) + evidence = tuple(edge.get("evidence_refs", [])) + candidate_rank = ( + 0 if evidence else 1, + -len(evidence), + -confidence_rank[edge["confidence"]], + -coverage_rank[edge["coverage"]], + evidence, + _sl_sha(edge), + ) + current = selected.get(key) + if current is None: + selected[key] = edge + continue + current_evidence = tuple(current.get("evidence_refs", [])) + current_rank = ( + 0 if current_evidence else 1, + -len(current_evidence), + -confidence_rank[current["confidence"]], + -coverage_rank[current["coverage"]], + current_evidence, + _sl_sha(current), + ) + if candidate_rank < current_rank: + selected[key] = edge + return sorted(selected.values(), key=lambda item: (item["from"], item["to"], item["type"], _sl_sha(item))) def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: @@ -1235,7 +1426,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: nodes.setdefault(node_id, {"node_id": node_id, "kind": _sl_node_kind(node_id), "coverage": {"state": "partial", "unknown": ["node_metadata"], "truncated": []}}) coverage_states = {edge["coverage"] for edge in all_edges} document_states = {document.get("coverage", {}).get("state", "unknown") for document in normalized} - node_states = {node.get("coverage", {}).get("state", "complete") for node in nodes.values() if node.get("kind") == "component"} + node_states = {node.get("coverage", {}).get("state", "complete") for node in nodes.values()} if "unknown" in coverage_states or "unknown" in document_states or "unknown" in node_states: coverage_state = "unknown" elif truncated or "partial" in coverage_states or "partial" in document_states or "partial" in node_states: @@ -1352,7 +1543,7 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: raise SBOMLineageError("lineage coverage lost document truncation metadata") edge_states = {edge["coverage"] for edge in checked_edges} document_states = {document["coverage"]["state"] for document in checked_documents} - node_states = {node["coverage"]["state"] for node in nodes if node.get("kind") == "component"} + node_states = {node["coverage"]["state"] for node in nodes} expected_state = "unknown" if "unknown" in edge_states or "unknown" in document_states or "unknown" in node_states else "partial" if truncated or "partial" in edge_states or "partial" in document_states or "partial" in node_states else "complete" expected_unknown = [] if expected_state == "complete" else ["external_lineage"] if state != expected_state or unknown != expected_unknown: @@ -1424,13 +1615,30 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 continue source, target = str(edge.get("from")), str(edge.get("to")) adjacency.setdefault(source, []).append((target, edge)) - adjacency.setdefault(target, []).append((source, edge)) + reverse = dict(edge) + reverse["from"], reverse["to"] = target, source + adjacency.setdefault(target, []).append((source, reverse)) found: dict[str, dict[str, Any]] = {} path_truncated = False + states_used = 0 + queued_states = 0 + budget_exhausted = False for start in matches: - queue = deque([(start, [], {start})]) + if queued_states >= _SL_MAX_QUERY_QUEUE: + path_truncated = True + truncated.append("query_queue") + break + queue = deque([(start, [])]) + queued_states += 1 + seen_nodes = {start} while queue: - current, path, visited = queue.popleft() + if states_used >= _SL_MAX_QUERY_STATES: + path_truncated = True + truncated.append("query_states") + budget_exhausted = True + break + current, path = queue.popleft() + states_used += 1 if current != start and _sl_node_kind(current) == "artifact": candidate = {"artifact_id": current, "path": path, "coverage": _sl_path_state(path), "confidence": _sl_path_confidence(path), "evidence_refs": sorted({ref for edge in path for ref in edge.get("evidence_refs", [])})} previous = found.get(current) @@ -1442,9 +1650,18 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 path_truncated = True continue for neighbor, edge in sorted(adjacency.get(current, []), key=lambda item: (item[0], item[1].get("type", ""))): - if neighbor in visited: + if neighbor in seen_nodes: + continue + seen_nodes.add(neighbor) + if len(queue) >= _SL_MAX_QUERY_QUEUE or queued_states >= _SL_MAX_QUERY_QUEUE: + path_truncated = True + truncated.append("query_queue") + budget_exhausted = True continue - queue.append((neighbor, path + [dict(edge)], visited | {neighbor})) + queue.append((neighbor, path + [dict(edge)])) + queued_states += 1 + if budget_exhausted: + break all_impacted = [found[key] for key in sorted(found)] if len(all_impacted) > limit: truncated.append("impacted_artifacts") @@ -1512,12 +1729,12 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] if not path or len(path) > _SL_MAX_PATH_LENGTH: raise SBOMLineageError("impacted_artifact.path is outside its bounds") checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] - if not any(set((edge["from"], edge["to"])) & set(matched_nodes) for edge in checked_path[:1]): + if checked_path[0]["from"] not in set(matched_nodes): raise SBOMLineageError("query path is not bound to a matched node") for first, second in zip(checked_path, checked_path[1:]): - if not set((first["from"], first["to"])) & set((second["from"], second["to"])): - raise SBOMLineageError("query path edges are not endpoint-bound") - if artifact_id not in {checked_path[-1]["from"], checked_path[-1]["to"]}: + if first["to"] != second["from"]: + raise SBOMLineageError("query path edges are not directed and contiguous") + if checked_path[-1]["to"] != artifact_id: raise SBOMLineageError("query path does not terminate at its artifact") coverage = item.get("coverage") confidence = item.get("confidence") diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 07013cf3..88c136f0 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import hashlib from pathlib import Path import pytest @@ -351,3 +352,132 @@ def test_query_numeric_bounds_and_complete_schema_are_fail_closed(): del invalid["components"][0]["coverage"]["unknown"] _reseal(invalid) assert perseus.verify_sbom_document(invalid)["valid"] is False + + +def test_unsafe_ids_properties_and_hash_algorithms_are_not_projected(): + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["bom-ref"] = "artifact:credential-token" + component["purl"] = "pkg:maven/example/pkg@1?api_key=RAW_PURL" + component["properties"] = [{"name": "vex-secret-property", "value": "RAW_PROPERTY_VALUE"}] + component["externalReferences"] = [{ + "type": "other", + "url": "https://source.invalid/?token=RAW_URL", + "hashes": [{"alg": "Bearer RAW_HASH_ALGORITHM", "content": "a" * 64}], + }] + document = perseus.ingest_sbom_document(payload, source_ref="artifact:secret-token-source") + encoded = json.dumps(document, sort_keys=True) + for secret in ("credential-token", "RAW_PURL", "RAW_PROPERTY_VALUE", "RAW_URL", "RAW_HASH_ALGORITHM", "secret-token-source"): + assert secret not in encoded + assert perseus.verify_sbom_document(document)["valid"] is True + + +def test_json_nesting_and_xml_dtd_are_rejected_as_sbom_errors(): + with pytest.raises(perseus.SBOMLineageError, match="nesting"): + perseus.ingest_sbom_document(b"[" * 20_000 + b"]" * 20_000) + with pytest.raises(perseus.SBOMLineageError, match="DTD|entity"): + perseus.ingest_sbom_document(b']>') + with pytest.raises(perseus.SBOMLineageError, match="nesting"): + perseus.ingest_sbom_document(b"" + b"" * (perseus._SL_MAX_XML_DEPTH + 1) + b"" * (perseus._SL_MAX_XML_DEPTH + 1) + b"") + with pytest.raises(perseus.SBOMLineageError, match="elements"): + perseus.ingest_sbom_document(b"" + b"" * (perseus._SL_MAX_XML_ELEMENTS + 1) + b"") + + +def test_cyclonedx_global_expansion_caps_produce_valid_truncated_documents(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"] = [ + {"type": "library", "bom-ref": f"pkg:generic/example-{i}@1", "name": f"example-{i}", "version": "1", "supplier": {"name": "Example"}} + for i in range(perseus._SL_MAX_COMPONENTS + 1) + ] + targets = [f"pkg:generic/example-{i}@1" for i in range(perseus._SL_MAX_RELATIONSHIPS)] + payload["dependencies"] = [{"ref": targets[0], "dependsOn": targets}] + document = perseus.ingest_sbom_document(payload) + assert len(document["components"]) <= perseus._SL_MAX_COMPONENTS + assert len(document["relationships"]) <= perseus._SL_MAX_RELATIONSHIPS + assert document["coverage"]["truncated"] + assert perseus.verify_sbom_document(document)["valid"] is True + + +def test_source_digest_must_match_raw_sbom_bytes(): + raw = _load("spdx-app.json") + expected = hashlib.sha256(raw).hexdigest() + document = perseus.ingest_sbom_document(raw, source_ref=f"sha256:{expected}") + assert document["source_ref"] == f"sha256:{expected}" + with pytest.raises(perseus.SBOMLineageError, match="bound"): + perseus.ingest_sbom_document(raw, source_ref="sha256:" + "0" * 64) + + forged = json.loads(json.dumps(document)) + forged["source_ref"] = "sha256:" + "1" * 64 + _reseal(forged) + with pytest.raises(perseus.SBOMLineageError, match="bound"): + perseus.build_sbom_lineage([forged]) + + public_document = perseus.ingest_sbom_document(raw, source_ref="artifact:fixture") + forged_digest = json.loads(json.dumps(public_document)) + forged_digest["document_sha256"] = "2" * 64 + _reseal(forged_digest) + with pytest.raises(perseus.SBOMLineageError, match="bound"): + perseus.build_sbom_lineage([forged_digest]) + + +def test_missing_document_metadata_and_external_nodes_are_partial(): + payload = json.loads(_load("spdx-app.json")) + payload.pop("name") + payload.pop("creationInfo") + document = perseus.ingest_sbom_document(payload) + assert document["coverage"]["state"] == "partial" + assert {"document_name", "created_at", "supplier"}.issubset(set(document["coverage"]["unknown"])) + lineage = perseus.build_sbom_lineage([document], edges=[ + {"from": "source:unresolved", "to": "SPDXRef-App", "type": "contains", "confidence": "high", "coverage": "complete"}, + ]) + assert lineage["coverage"]["state"] == "partial" + assert perseus.verify_sbom_lineage(lineage)["valid"] is True + + +def test_query_verifier_requires_directed_contiguous_path_edges(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + result = perseus.query_sbom_lineage(perseus.build_sbom_lineage([document], edges=_lineage_edges()), "CVE-2021-44228") + invalid = json.loads(json.dumps(result)) + path = invalid["impacted_artifacts"][0]["path"] + path[1]["from"], path[1]["to"] = path[1]["to"], path[1]["from"] + _reseal(invalid) + assert perseus.verify_sbom_lineage_query(invalid)["valid"] is False + + +def test_query_bfs_has_global_state_and_queue_budgets(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + edges = [] + hubs = [f"build:hub-{i}" for i in range(20)] + for source_index in range(200): + source = f"source:needle-{source_index}" + for hub in hubs: + edges.append({"from": source, "to": hub, "type": "routes", "confidence": "high", "coverage": "complete"}) + edges.extend({"from": hub, "to": "artifact:budget-target", "type": "generates", "confidence": "high", "coverage": "complete"} for hub in hubs) + lineage = perseus.build_sbom_lineage([document], edges=edges) + result = perseus.query_sbom_lineage(lineage, "needle", limit=32) + assert "query_queue" in result["coverage"]["truncated"] or "query_states" in result["coverage"]["truncated"] + assert result["status"] != "complete" + + +def test_conflicting_same_key_edges_are_deterministic_and_tie_break_evidence(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + edges = [ + {"from": "source:stable", "to": "artifact:stable", "type": "generates", "confidence": "high", "coverage": "complete"}, + {"from": "source:stable", "to": "artifact:stable", "type": "generates", "confidence": "unknown", "coverage": "unknown", "evidence_refs": ["ledger:strong"]}, + ] + first = perseus.build_sbom_lineage([document], edges=edges) + second = perseus.build_sbom_lineage([document], edges=list(reversed(edges))) + assert first["lineage_digest"] == second["lineage_digest"] + selected = next(edge for edge in first["edges"] if edge["from"] == "source:stable") + assert selected["evidence_refs"] == ["ledger:strong"] + + +def test_cyclonedx_purl_qualifiers_are_valid_component_ids(): + payload = json.loads(_load("cyclonedx-app.json")) + qualified = "pkg:maven/org.example/example@1.0?classifier=sources&repository_url=https%3A%2F%2Frepo.example" + payload["components"][0]["bom-ref"] = qualified + payload["components"][0]["purl"] = qualified + payload["dependencies"][0]["dependsOn"] = [qualified] + document = perseus.ingest_sbom_document(payload) + assert any(component["component_id"] == qualified for component in document["components"]) + assert perseus.verify_sbom_document(document)["valid"] is True From 17f53e2481540682aef3aff9df2a389a63148c7e Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 16:52:38 +0000 Subject: [PATCH 03/40] fix(sbom): close lineage validation and rebinding gaps --- perseus.py | 218 ++++++++++++++++++++++++++++++------ src/perseus/cli.py | 1 + src/perseus/sbom_lineage.py | 215 +++++++++++++++++++++++++++++------ tests/test_sbom_lineage.py | 129 +++++++++++++++++++++ 4 files changed, 490 insertions(+), 73 deletions(-) diff --git a/perseus.py b/perseus.py index 5e8934ed..f0cb36a8 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "a599b56-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "8610f04-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44448,6 +44448,7 @@ def cmd_code_map(args, cfg) -> int: import json import re import xml.etree.ElementTree as _sl_et +from urllib.parse import unquote from collections import deque from pathlib import Path from typing import Any, Mapping @@ -44513,6 +44514,13 @@ def cmd_code_map(args, cfg) -> int: } +# In-process provenance for normalized projections. A projection digest alone +# is caller-recomputable; only a projection produced by raw ingestion (or one +# explicitly re-verified against raw bytes) is accepted by lineage builders. +_SL_INGESTED_PROVENANCE: dict[str, tuple[str, str]] = {} +_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str]] = {} + + class SBOMLineageError(ValueError): """Raised when an SBOM or lineage projection cannot be verified.""" @@ -44525,19 +44533,33 @@ def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() -def _sl_ingestion_sha(unsigned: Mapping[str, Any]) -> str: - return _sl_sha({"domain": "perseus-sbom-ingestion", "document_sha256": unsigned.get("document_sha256"), "projection": unsigned}) +def _sl_ingestion_sha(unsigned: Mapping[str, Any], raw_bytes: bytes | None = None) -> str: + raw_digest = hashlib.sha256(raw_bytes).hexdigest() if raw_bytes is not None else unsigned.get("document_sha256") + return _sl_sha({"domain": "perseus-sbom-ingestion", "raw_sha256": raw_digest, "projection": unsigned}) def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" - authority = value.split("://", 1)[1].split("/", 1)[0] if "://" in value else "" - normalized = re.sub(r"[^a-z0-9]+", "_", value.casefold()) - return bool( - _SL_SENSITIVE_REFERENCE_RE.search(value) - or (authority and "@" in authority) - or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) - ) + candidates = [value] + decoded = value + # Decode a small, bounded number of layers so encoded URL/PURL userinfo + # and query keys cannot evade the credential checks below. + for _ in range(3): + next_decoded = unquote(decoded) + if next_decoded == decoded: + break + candidates.append(next_decoded) + decoded = next_decoded + for candidate in candidates: + authority = candidate.split("://", 1)[1].split("/", 1)[0] if "://" in candidate else "" + normalized = re.sub(r"[^a-z0-9]+", "_", candidate.casefold()) + if ( + _SL_SENSITIVE_REFERENCE_RE.search(candidate) + or (authority and "@" in authority) + or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) + ): + return True + return False def _sl_truncated(truncated: list[str] | None, name: str) -> None: @@ -44584,7 +44606,10 @@ def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512 def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: - text = _sl_text(value if value is not None else fallback, field, required=True, limit=256) + candidate = fallback if value is None else value + if not isinstance(candidate, str): + raise SBOMLineageError(f"{field} must be a string") + text = _sl_text(candidate, field, required=True, limit=256) if _sl_sensitive(text): digest = hashlib.sha256(text.encode("utf-8")).hexdigest() prefix = text.split(":", 1)[0] @@ -44609,6 +44634,10 @@ def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" text = _sl_text(value, "source_ref", required=True, limit=512) raw_digest = hashlib.sha256(raw_bytes).hexdigest() + if text.startswith("sha256:source-ref:"): + if not re.fullmatch(r"sha256:source-ref:[0-9a-fA-F]{64}", text): + raise SBOMLineageError("source_ref sanitized digest is malformed") + return text.lower() if text.startswith("sha256:"): if not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text) or text[7:].casefold() != raw_digest: raise SBOMLineageError("source_ref digest is not bound to the SBOM bytes") @@ -45066,7 +45095,11 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "dangling_relationships": dangling, }, } - unsigned["ingestion_digest"] = _sl_ingestion_sha(unsigned) + unsigned["ingestion_digest"] = _sl_ingestion_sha(unsigned, raw_bytes) + _SL_INGESTED_PROVENANCE[unsigned["ingestion_digest"]] = ( + unsigned["document_sha256"], + _sl_json(unsigned), + ) return unsigned @@ -45419,6 +45452,20 @@ def _sl_validate_json_depth(raw_bytes: bytes) -> None: depth = max(0, depth - 1) +def _sl_load_json_bounded(source: Path | bytes, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: + """Load JSON through the shared byte and nesting bounds.""" + raw_bytes = _sl_read_bounded(source) if isinstance(source, Path) else bytes(source) + _sl_validate_json_depth(raw_bytes) + try: + return json.loads(raw_bytes.decode("utf-8")), raw_bytes + except RecursionError as exc: + raise SBOMLineageError("SBOM JSON nesting is too deep") from exc + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + if allow_non_json: + return None, raw_bytes + raise SBOMLineageError("SBOM JSON is invalid") from exc + + def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: if re.search(rb" dict[str, Any]: +def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | None = None) -> dict[str, Any]: required = { "schema_version", "format", "spec_version", "document_id", "document_name", "document_sha256", "source_ref", "created_at", "supplier", "components", @@ -45709,9 +45756,16 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: supplied = document.get("ingestion_digest") unsigned = dict(document) unsigned.pop("ingestion_digest", None) - if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or supplied not in {_sl_ingestion_sha(unsigned), _sl_sha(unsigned)}: - raise SBOMLineageError("normalized SBOM ingestion digest mismatch") - legacy_ingestion_digest = supplied == _sl_sha(unsigned) + expected_raw_sha = hashlib.sha256(raw_bytes).hexdigest() if raw_bytes is not None else unsigned.get("document_sha256") + expected_ingestion = _sl_ingestion_sha(unsigned, raw_bytes) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or supplied != expected_ingestion: + raise SBOMLineageError("normalized SBOM ingestion digest mismatch; unsafe or source-bound projection required") + if raw_bytes is not None and unsigned.get("document_sha256") != expected_raw_sha: + raise SBOMLineageError("normalized SBOM document digest is not bound to raw ingestion bytes") + if raw_bytes is None: + provenance = _SL_INGESTED_PROVENANCE.get(supplied) + if provenance is None or provenance != (unsigned.get("document_sha256"), _sl_json(document)): + raise SBOMLineageError("normalized SBOM is not bound to raw ingestion bytes/source digest") fmt = document.get("format") if fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") @@ -45759,14 +45813,22 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: ) if coverage != expected: raise SBOMLineageError("document coverage is inconsistent with its contents") - if legacy_ingestion_digest: - raise SBOMLineageError("normalized SBOM ingestion digest is not bound to document bytes") return dict(document) -def verify_sbom_document(document: Mapping[str, Any]) -> dict[str, Any]: +def _sl_rebind_document(document: Mapping[str, Any], raw_document: Any) -> dict[str, Any]: + """Re-verify a persisted normalized projection against its raw source bytes.""" + _, raw_bytes = _sl_payload(raw_document) + checked = _sl_validate_document(document, raw_bytes=raw_bytes) + expected = ingest_sbom_document(raw_bytes, source_ref=document.get("source_ref", "")) + if _sl_json(expected) != _sl_json(dict(document)): + raise SBOMLineageError("normalized SBOM projection is not bound to raw ingestion bytes") + return checked + + +def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = None) -> dict[str, Any]: try: - checked = _sl_validate_document(document) + checked = _sl_validate_document(document) if raw_document is None else _sl_rebind_document(document, raw_document) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} @@ -45790,9 +45852,9 @@ def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list )) confidence_rank = {"high": 3, "medium": 2, "low": 1, "unknown": 0} coverage_rank = {"complete": 2, "partial": 1, "unknown": 0} - selected: dict[tuple[str, str, str], dict[str, Any]] = {} + selected: dict[tuple[str, str], dict[str, Any]] = {} for edge in result: - key = (edge["from"], edge["to"], edge["type"]) + key = (edge["from"], edge["to"]) evidence = tuple(edge.get("evidence_refs", [])) candidate_rank = ( 0 if evidence else 1, @@ -45820,18 +45882,28 @@ def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list return sorted(selected.values(), key=lambda item: (item["from"], item["to"], item["type"], _sl_sha(item))) -def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: +def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any = None) -> dict[str, Any]: """Build a deterministic graph from normalized or raw SBOM documents.""" if not isinstance(documents, (list, tuple)) or not documents: raise SBOMLineageError("at least one SBOM document is required") + rebound_raw: list[Any] | None = None + if raw_documents is not None: + if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: + raise SBOMLineageError("raw_documents must match the document list within its bound") + rebound_raw = list(raw_documents) truncated: list[str] = [] if len(documents) > _SL_MAX_DOCUMENTS: truncated.append("documents") normalized = [] - for document in documents[:_SL_MAX_DOCUMENTS]: + for index, document in enumerate(documents[:_SL_MAX_DOCUMENTS]): if isinstance(document, Mapping) and document.get("schema_version") == _SL_SCHEMA: - normalized.append(_sl_validate_document(document)) + if rebound_raw is None: + normalized.append(_sl_validate_document(document)) + else: + normalized.append(_sl_rebind_document(document, rebound_raw[index])) else: + if rebound_raw is not None: + raise SBOMLineageError("raw_documents may only rebind normalized SBOM projections") normalized.append(ingest_sbom_document(document)) nodes: dict[str, dict[str, Any]] = {} component_fingerprints: dict[str, str] = {} @@ -45844,6 +45916,8 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: raise SBOMLineageError(f"conflicting duplicate component ID: {component_id}") if component_id in component_fingerprints: continue + if len(nodes) >= _SL_MAX_NODES: + raise SBOMLineageError("lineage nodes exceed its global bound") component_fingerprints[component_id] = fingerprint node = dict(component) node["node_id"] = component_id @@ -45859,6 +45933,8 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: all_edges = _sl_lineage_edges(native_edges + external_edges, truncated=truncated) for edge in all_edges: for node_id in (edge["from"], edge["to"]): + if node_id not in nodes and len(nodes) >= _SL_MAX_NODES: + raise SBOMLineageError("lineage nodes exceed its global bound") nodes.setdefault(node_id, {"node_id": node_id, "kind": _sl_node_kind(node_id), "coverage": {"state": "partial", "unknown": ["node_metadata"], "truncated": []}}) coverage_states = {edge["coverage"] for edge in all_edges} document_states = {document.get("coverage", {}).get("state", "unknown") for document in normalized} @@ -45877,6 +45953,10 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: "coverage": {"state": coverage_state, "unknown": [] if coverage_state == "complete" else ["external_lineage"], "truncated": sorted(set(truncated))}, } body["lineage_digest"] = _sl_sha(body) + _SL_LINEAGE_PROVENANCE[body["lineage_digest"]] = ( + _sl_json(body["nodes"]), + _sl_json(body["edges"]), + ) return body @@ -46121,6 +46201,9 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 impact_status = "not_established" unsigned: dict[str, Any] = { "schema_version": _SL_QUERY_SCHEMA, + "lineage_digest": loaded["lineage_digest"], + "lineage_nodes": loaded["nodes"], + "lineage_edges": loaded["edges"], "query": query_text, "matched_nodes": matches, "impacted_artifacts": impacted, @@ -46132,8 +46215,8 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 return unsigned -def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any]: - required = {"schema_version", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} +def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: + required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") supplied = query_result.get("query_digest") @@ -46141,11 +46224,52 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] unsigned.pop("query_digest", None) if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: raise SBOMLineageError("query digest mismatch") + lineage_digest = query_result.get("lineage_digest") + if not isinstance(lineage_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", lineage_digest): + raise SBOMLineageError("query lineage digest is invalid") + lineage_nodes = _sl_list(query_result.get("lineage_nodes"), "query.lineage_nodes") + lineage_edges = _sl_list(query_result.get("lineage_edges"), "query.lineage_edges") + if len(lineage_nodes) > _SL_MAX_NODES or len(lineage_edges) > _SL_MAX_EDGES: + raise SBOMLineageError("query authoritative lineage exceeds its bound") + authority = _SL_LINEAGE_PROVENANCE.get(lineage_digest) + if authoritative_lineage is not None: + loaded_authority = _sl_loaded_lineage(authoritative_lineage) + if lineage_digest != loaded_authority["lineage_digest"]: + raise SBOMLineageError("query result lineage digest does not match the authoritative lineage") + authority = (_sl_json(loaded_authority["nodes"]), _sl_json(loaded_authority["edges"])) + if authority is None or authority != (_sl_json(lineage_nodes), _sl_json(lineage_edges)): + raise SBOMLineageError("query result is not bound to the authoritative lineage digest and nodes/edges") + authority_node_ids: set[str] = set() + for node in lineage_nodes: + if not isinstance(node, Mapping): + raise SBOMLineageError("query authoritative node must be an object") + _sl_require_keys(node, {"node_id", "kind", "coverage"}, {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "document_sha256"}, "query.authoritative_node") + node_id = _sl_strict_text(node.get("node_id"), "query.authoritative_node.node_id", limit=256) + kind = _sl_strict_text(node.get("kind"), "query.authoritative_node.kind", limit=32) + assert node_id is not None and kind is not None + _sl_id(node_id, "query.authoritative_node.node_id") + if kind != _sl_node_kind(node_id) or node_id in authority_node_ids: + raise SBOMLineageError("query authoritative node identity is invalid") + authority_node_ids.add(node_id) + full_component = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} + if full_component.issubset(set(node)): + _sl_validate_component({key: node[key] for key in full_component}) + _sl_validate_node_coverage(node.get("coverage"), component=True) + else: + _sl_validate_node_coverage(node.get("coverage"), component=False) + authority_edges: list[dict[str, Any]] = [] + for edge in lineage_edges: + checked_edge = _sl_validate_relationship(edge, field="query.authoritative_edge") + if checked_edge["from"] not in authority_node_ids or checked_edge["to"] not in authority_node_ids: + raise SBOMLineageError("query authoritative edge endpoint is not bound to a node") + authority_edges.append(checked_edge) query = _sl_strict_text(query_result.get("query"), "query", limit=512) assert query is not None matched_nodes = _sl_string_list(query_result.get("matched_nodes"), "matched_nodes", maximum=_SL_MAX_QUERY_MATCHES) for node_id in matched_nodes: _sl_id(node_id, "matched_node") + if not set(matched_nodes).issubset(authority_node_ids): + raise SBOMLineageError("query matched nodes are not bound to the authoritative lineage") impacted = _sl_list(query_result.get("impacted_artifacts"), "impacted_artifacts") if len(impacted) > _SL_MAX_QUERY_RESULTS: raise SBOMLineageError("impacted_artifacts exceeds its bound") @@ -46158,6 +46282,8 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] _sl_id(artifact_id, "artifact_id") if _sl_node_kind(artifact_id) != "artifact": raise SBOMLineageError("impacted artifact ID is not an artifact") + if artifact_id not in authority_node_ids: + raise SBOMLineageError("impacted artifact is not bound to the authoritative lineage") if impacted_ids and artifact_id <= impacted_ids[-1]: raise SBOMLineageError("impacted_artifacts must be unique and sorted") impacted_ids.append(artifact_id) @@ -46165,6 +46291,11 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] if not path or len(path) > _SL_MAX_PATH_LENGTH: raise SBOMLineageError("impacted_artifact.path is outside its bounds") checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] + for path_edge in checked_path: + reversed_edge = dict(path_edge) + reversed_edge["from"], reversed_edge["to"] = path_edge["to"], path_edge["from"] + if not any(path_edge == authority_edge or reversed_edge == authority_edge for authority_edge in authority_edges): + raise SBOMLineageError("query path edge is not bound to the authoritative lineage") if checked_path[0]["from"] not in set(matched_nodes): raise SBOMLineageError("query path is not bound to a matched node") for first, second in zip(checked_path, checked_path[1:]): @@ -46218,9 +46349,9 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] return dict(query_result) -def verify_sbom_lineage_query(query_result: Mapping[str, Any]) -> dict[str, Any]: +def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: try: - checked = _sl_validate_query_result(query_result) + checked = _sl_validate_query_result(query_result, authoritative_lineage) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} @@ -46236,16 +46367,28 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: elif command == "merge": documents = [] for path in args.documents: - raw_bytes = _sl_read_bounded(Path(path)) - try: - payload = json.loads(raw_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - payload = None + payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) - edges = json.loads(_sl_read_bounded(Path(args.edges)).decode("utf-8")) if getattr(args, "edges", None) else [] - document = build_sbom_lineage(documents, edges=edges) + raw_documents = getattr(args, "raw_documents", None) + raw_inputs = None + if raw_documents is not None: + if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents): + raise SBOMLineageError("raw_documents must contain one raw source per document") + raw_inputs = [] + for raw_path in raw_documents: + _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) + raw_inputs.append(raw_bytes) + if getattr(args, "edges", None): + edges, _ = _sl_load_json_bounded(Path(args.edges)) + if not isinstance(edges, list): + raise SBOMLineageError("lineage edges JSON must be a list") + else: + edges = [] + document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) elif command == "query": - lineage = json.loads(_sl_read_bounded(Path(args.lineage)).decode("utf-8")) + lineage, _ = _sl_load_json_bounded(Path(args.lineage)) + if not isinstance(lineage, Mapping): + raise SBOMLineageError("lineage JSON root must be an object") document = query_sbom_lineage(lineage, args.component, limit=getattr(args, "limit", 32)) else: raise SBOMLineageError("command must be ingest, merge, or query") @@ -47116,6 +47259,7 @@ def main(): p_sbom_ingest.add_argument("--json", action="store_true", help="Print machine-readable JSON") p_sbom_merge = sbom_sub.add_parser("merge", help="Build a queryable lineage graph from normalized SBOM documents") p_sbom_merge.add_argument("documents", nargs="+", help="SBOM document paths") + p_sbom_merge.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to normalized documents") p_sbom_merge.add_argument("--edges", default=None, help="Optional JSON file of source/build/artifact/deployment edges") p_sbom_merge.add_argument("--output", "-o", default=None, help="Write the lineage graph to a JSON file") p_sbom_merge.add_argument("--json", action="store_true", help="Print machine-readable JSON") diff --git a/src/perseus/cli.py b/src/perseus/cli.py index bb50b781..a79864ac 100644 --- a/src/perseus/cli.py +++ b/src/perseus/cli.py @@ -162,6 +162,7 @@ def main(): p_sbom_ingest.add_argument("--json", action="store_true", help="Print machine-readable JSON") p_sbom_merge = sbom_sub.add_parser("merge", help="Build a queryable lineage graph from normalized SBOM documents") p_sbom_merge.add_argument("documents", nargs="+", help="SBOM document paths") + p_sbom_merge.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to normalized documents") p_sbom_merge.add_argument("--edges", default=None, help="Optional JSON file of source/build/artifact/deployment edges") p_sbom_merge.add_argument("--output", "-o", default=None, help="Write the lineage graph to a JSON file") p_sbom_merge.add_argument("--json", action="store_true", help="Print machine-readable JSON") diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 6b371df6..32f11b1f 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -12,6 +12,7 @@ import json import re import xml.etree.ElementTree as _sl_et +from urllib.parse import unquote from collections import deque from pathlib import Path from typing import Any, Mapping @@ -77,6 +78,13 @@ } +# In-process provenance for normalized projections. A projection digest alone +# is caller-recomputable; only a projection produced by raw ingestion (or one +# explicitly re-verified against raw bytes) is accepted by lineage builders. +_SL_INGESTED_PROVENANCE: dict[str, tuple[str, str]] = {} +_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str]] = {} + + class SBOMLineageError(ValueError): """Raised when an SBOM or lineage projection cannot be verified.""" @@ -89,19 +97,33 @@ def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() -def _sl_ingestion_sha(unsigned: Mapping[str, Any]) -> str: - return _sl_sha({"domain": "perseus-sbom-ingestion", "document_sha256": unsigned.get("document_sha256"), "projection": unsigned}) +def _sl_ingestion_sha(unsigned: Mapping[str, Any], raw_bytes: bytes | None = None) -> str: + raw_digest = hashlib.sha256(raw_bytes).hexdigest() if raw_bytes is not None else unsigned.get("document_sha256") + return _sl_sha({"domain": "perseus-sbom-ingestion", "raw_sha256": raw_digest, "projection": unsigned}) def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" - authority = value.split("://", 1)[1].split("/", 1)[0] if "://" in value else "" - normalized = re.sub(r"[^a-z0-9]+", "_", value.casefold()) - return bool( - _SL_SENSITIVE_REFERENCE_RE.search(value) - or (authority and "@" in authority) - or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) - ) + candidates = [value] + decoded = value + # Decode a small, bounded number of layers so encoded URL/PURL userinfo + # and query keys cannot evade the credential checks below. + for _ in range(3): + next_decoded = unquote(decoded) + if next_decoded == decoded: + break + candidates.append(next_decoded) + decoded = next_decoded + for candidate in candidates: + authority = candidate.split("://", 1)[1].split("/", 1)[0] if "://" in candidate else "" + normalized = re.sub(r"[^a-z0-9]+", "_", candidate.casefold()) + if ( + _SL_SENSITIVE_REFERENCE_RE.search(candidate) + or (authority and "@" in authority) + or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) + ): + return True + return False def _sl_truncated(truncated: list[str] | None, name: str) -> None: @@ -148,7 +170,10 @@ def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512 def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: - text = _sl_text(value if value is not None else fallback, field, required=True, limit=256) + candidate = fallback if value is None else value + if not isinstance(candidate, str): + raise SBOMLineageError(f"{field} must be a string") + text = _sl_text(candidate, field, required=True, limit=256) if _sl_sensitive(text): digest = hashlib.sha256(text.encode("utf-8")).hexdigest() prefix = text.split(":", 1)[0] @@ -173,6 +198,10 @@ def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" text = _sl_text(value, "source_ref", required=True, limit=512) raw_digest = hashlib.sha256(raw_bytes).hexdigest() + if text.startswith("sha256:source-ref:"): + if not re.fullmatch(r"sha256:source-ref:[0-9a-fA-F]{64}", text): + raise SBOMLineageError("source_ref sanitized digest is malformed") + return text.lower() if text.startswith("sha256:"): if not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", text) or text[7:].casefold() != raw_digest: raise SBOMLineageError("source_ref digest is not bound to the SBOM bytes") @@ -630,7 +659,11 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "dangling_relationships": dangling, }, } - unsigned["ingestion_digest"] = _sl_ingestion_sha(unsigned) + unsigned["ingestion_digest"] = _sl_ingestion_sha(unsigned, raw_bytes) + _SL_INGESTED_PROVENANCE[unsigned["ingestion_digest"]] = ( + unsigned["document_sha256"], + _sl_json(unsigned), + ) return unsigned @@ -983,6 +1016,20 @@ def _sl_validate_json_depth(raw_bytes: bytes) -> None: depth = max(0, depth - 1) +def _sl_load_json_bounded(source: Path | bytes, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: + """Load JSON through the shared byte and nesting bounds.""" + raw_bytes = _sl_read_bounded(source) if isinstance(source, Path) else bytes(source) + _sl_validate_json_depth(raw_bytes) + try: + return json.loads(raw_bytes.decode("utf-8")), raw_bytes + except RecursionError as exc: + raise SBOMLineageError("SBOM JSON nesting is too deep") from exc + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + if allow_non_json: + return None, raw_bytes + raise SBOMLineageError("SBOM JSON is invalid") from exc + + def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: if re.search(rb" dict[str, Any]: +def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | None = None) -> dict[str, Any]: required = { "schema_version", "format", "spec_version", "document_id", "document_name", "document_sha256", "source_ref", "created_at", "supplier", "components", @@ -1273,9 +1320,16 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: supplied = document.get("ingestion_digest") unsigned = dict(document) unsigned.pop("ingestion_digest", None) - if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or supplied not in {_sl_ingestion_sha(unsigned), _sl_sha(unsigned)}: - raise SBOMLineageError("normalized SBOM ingestion digest mismatch") - legacy_ingestion_digest = supplied == _sl_sha(unsigned) + expected_raw_sha = hashlib.sha256(raw_bytes).hexdigest() if raw_bytes is not None else unsigned.get("document_sha256") + expected_ingestion = _sl_ingestion_sha(unsigned, raw_bytes) + if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or supplied != expected_ingestion: + raise SBOMLineageError("normalized SBOM ingestion digest mismatch; unsafe or source-bound projection required") + if raw_bytes is not None and unsigned.get("document_sha256") != expected_raw_sha: + raise SBOMLineageError("normalized SBOM document digest is not bound to raw ingestion bytes") + if raw_bytes is None: + provenance = _SL_INGESTED_PROVENANCE.get(supplied) + if provenance is None or provenance != (unsigned.get("document_sha256"), _sl_json(document)): + raise SBOMLineageError("normalized SBOM is not bound to raw ingestion bytes/source digest") fmt = document.get("format") if fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") @@ -1323,14 +1377,22 @@ def _sl_validate_document(document: Mapping[str, Any]) -> dict[str, Any]: ) if coverage != expected: raise SBOMLineageError("document coverage is inconsistent with its contents") - if legacy_ingestion_digest: - raise SBOMLineageError("normalized SBOM ingestion digest is not bound to document bytes") return dict(document) -def verify_sbom_document(document: Mapping[str, Any]) -> dict[str, Any]: +def _sl_rebind_document(document: Mapping[str, Any], raw_document: Any) -> dict[str, Any]: + """Re-verify a persisted normalized projection against its raw source bytes.""" + _, raw_bytes = _sl_payload(raw_document) + checked = _sl_validate_document(document, raw_bytes=raw_bytes) + expected = ingest_sbom_document(raw_bytes, source_ref=document.get("source_ref", "")) + if _sl_json(expected) != _sl_json(dict(document)): + raise SBOMLineageError("normalized SBOM projection is not bound to raw ingestion bytes") + return checked + + +def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = None) -> dict[str, Any]: try: - checked = _sl_validate_document(document) + checked = _sl_validate_document(document) if raw_document is None else _sl_rebind_document(document, raw_document) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} @@ -1354,9 +1416,9 @@ def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list )) confidence_rank = {"high": 3, "medium": 2, "low": 1, "unknown": 0} coverage_rank = {"complete": 2, "partial": 1, "unknown": 0} - selected: dict[tuple[str, str, str], dict[str, Any]] = {} + selected: dict[tuple[str, str], dict[str, Any]] = {} for edge in result: - key = (edge["from"], edge["to"], edge["type"]) + key = (edge["from"], edge["to"]) evidence = tuple(edge.get("evidence_refs", [])) candidate_rank = ( 0 if evidence else 1, @@ -1384,18 +1446,28 @@ def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list return sorted(selected.values(), key=lambda item: (item["from"], item["to"], item["type"], _sl_sha(item))) -def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: +def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any = None) -> dict[str, Any]: """Build a deterministic graph from normalized or raw SBOM documents.""" if not isinstance(documents, (list, tuple)) or not documents: raise SBOMLineageError("at least one SBOM document is required") + rebound_raw: list[Any] | None = None + if raw_documents is not None: + if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: + raise SBOMLineageError("raw_documents must match the document list within its bound") + rebound_raw = list(raw_documents) truncated: list[str] = [] if len(documents) > _SL_MAX_DOCUMENTS: truncated.append("documents") normalized = [] - for document in documents[:_SL_MAX_DOCUMENTS]: + for index, document in enumerate(documents[:_SL_MAX_DOCUMENTS]): if isinstance(document, Mapping) and document.get("schema_version") == _SL_SCHEMA: - normalized.append(_sl_validate_document(document)) + if rebound_raw is None: + normalized.append(_sl_validate_document(document)) + else: + normalized.append(_sl_rebind_document(document, rebound_raw[index])) else: + if rebound_raw is not None: + raise SBOMLineageError("raw_documents may only rebind normalized SBOM projections") normalized.append(ingest_sbom_document(document)) nodes: dict[str, dict[str, Any]] = {} component_fingerprints: dict[str, str] = {} @@ -1408,6 +1480,8 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: raise SBOMLineageError(f"conflicting duplicate component ID: {component_id}") if component_id in component_fingerprints: continue + if len(nodes) >= _SL_MAX_NODES: + raise SBOMLineageError("lineage nodes exceed its global bound") component_fingerprints[component_id] = fingerprint node = dict(component) node["node_id"] = component_id @@ -1423,6 +1497,8 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: all_edges = _sl_lineage_edges(native_edges + external_edges, truncated=truncated) for edge in all_edges: for node_id in (edge["from"], edge["to"]): + if node_id not in nodes and len(nodes) >= _SL_MAX_NODES: + raise SBOMLineageError("lineage nodes exceed its global bound") nodes.setdefault(node_id, {"node_id": node_id, "kind": _sl_node_kind(node_id), "coverage": {"state": "partial", "unknown": ["node_metadata"], "truncated": []}}) coverage_states = {edge["coverage"] for edge in all_edges} document_states = {document.get("coverage", {}).get("state", "unknown") for document in normalized} @@ -1441,6 +1517,10 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None) -> dict[str, Any]: "coverage": {"state": coverage_state, "unknown": [] if coverage_state == "complete" else ["external_lineage"], "truncated": sorted(set(truncated))}, } body["lineage_digest"] = _sl_sha(body) + _SL_LINEAGE_PROVENANCE[body["lineage_digest"]] = ( + _sl_json(body["nodes"]), + _sl_json(body["edges"]), + ) return body @@ -1685,6 +1765,9 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 impact_status = "not_established" unsigned: dict[str, Any] = { "schema_version": _SL_QUERY_SCHEMA, + "lineage_digest": loaded["lineage_digest"], + "lineage_nodes": loaded["nodes"], + "lineage_edges": loaded["edges"], "query": query_text, "matched_nodes": matches, "impacted_artifacts": impacted, @@ -1696,8 +1779,8 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 return unsigned -def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any]: - required = {"schema_version", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} +def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: + required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") supplied = query_result.get("query_digest") @@ -1705,11 +1788,52 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] unsigned.pop("query_digest", None) if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: raise SBOMLineageError("query digest mismatch") + lineage_digest = query_result.get("lineage_digest") + if not isinstance(lineage_digest, str) or not re.fullmatch(r"[0-9a-f]{64}", lineage_digest): + raise SBOMLineageError("query lineage digest is invalid") + lineage_nodes = _sl_list(query_result.get("lineage_nodes"), "query.lineage_nodes") + lineage_edges = _sl_list(query_result.get("lineage_edges"), "query.lineage_edges") + if len(lineage_nodes) > _SL_MAX_NODES or len(lineage_edges) > _SL_MAX_EDGES: + raise SBOMLineageError("query authoritative lineage exceeds its bound") + authority = _SL_LINEAGE_PROVENANCE.get(lineage_digest) + if authoritative_lineage is not None: + loaded_authority = _sl_loaded_lineage(authoritative_lineage) + if lineage_digest != loaded_authority["lineage_digest"]: + raise SBOMLineageError("query result lineage digest does not match the authoritative lineage") + authority = (_sl_json(loaded_authority["nodes"]), _sl_json(loaded_authority["edges"])) + if authority is None or authority != (_sl_json(lineage_nodes), _sl_json(lineage_edges)): + raise SBOMLineageError("query result is not bound to the authoritative lineage digest and nodes/edges") + authority_node_ids: set[str] = set() + for node in lineage_nodes: + if not isinstance(node, Mapping): + raise SBOMLineageError("query authoritative node must be an object") + _sl_require_keys(node, {"node_id", "kind", "coverage"}, {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "document_sha256"}, "query.authoritative_node") + node_id = _sl_strict_text(node.get("node_id"), "query.authoritative_node.node_id", limit=256) + kind = _sl_strict_text(node.get("kind"), "query.authoritative_node.kind", limit=32) + assert node_id is not None and kind is not None + _sl_id(node_id, "query.authoritative_node.node_id") + if kind != _sl_node_kind(node_id) or node_id in authority_node_ids: + raise SBOMLineageError("query authoritative node identity is invalid") + authority_node_ids.add(node_id) + full_component = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} + if full_component.issubset(set(node)): + _sl_validate_component({key: node[key] for key in full_component}) + _sl_validate_node_coverage(node.get("coverage"), component=True) + else: + _sl_validate_node_coverage(node.get("coverage"), component=False) + authority_edges: list[dict[str, Any]] = [] + for edge in lineage_edges: + checked_edge = _sl_validate_relationship(edge, field="query.authoritative_edge") + if checked_edge["from"] not in authority_node_ids or checked_edge["to"] not in authority_node_ids: + raise SBOMLineageError("query authoritative edge endpoint is not bound to a node") + authority_edges.append(checked_edge) query = _sl_strict_text(query_result.get("query"), "query", limit=512) assert query is not None matched_nodes = _sl_string_list(query_result.get("matched_nodes"), "matched_nodes", maximum=_SL_MAX_QUERY_MATCHES) for node_id in matched_nodes: _sl_id(node_id, "matched_node") + if not set(matched_nodes).issubset(authority_node_ids): + raise SBOMLineageError("query matched nodes are not bound to the authoritative lineage") impacted = _sl_list(query_result.get("impacted_artifacts"), "impacted_artifacts") if len(impacted) > _SL_MAX_QUERY_RESULTS: raise SBOMLineageError("impacted_artifacts exceeds its bound") @@ -1722,6 +1846,8 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] _sl_id(artifact_id, "artifact_id") if _sl_node_kind(artifact_id) != "artifact": raise SBOMLineageError("impacted artifact ID is not an artifact") + if artifact_id not in authority_node_ids: + raise SBOMLineageError("impacted artifact is not bound to the authoritative lineage") if impacted_ids and artifact_id <= impacted_ids[-1]: raise SBOMLineageError("impacted_artifacts must be unique and sorted") impacted_ids.append(artifact_id) @@ -1729,6 +1855,11 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] if not path or len(path) > _SL_MAX_PATH_LENGTH: raise SBOMLineageError("impacted_artifact.path is outside its bounds") checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] + for path_edge in checked_path: + reversed_edge = dict(path_edge) + reversed_edge["from"], reversed_edge["to"] = path_edge["to"], path_edge["from"] + if not any(path_edge == authority_edge or reversed_edge == authority_edge for authority_edge in authority_edges): + raise SBOMLineageError("query path edge is not bound to the authoritative lineage") if checked_path[0]["from"] not in set(matched_nodes): raise SBOMLineageError("query path is not bound to a matched node") for first, second in zip(checked_path, checked_path[1:]): @@ -1782,9 +1913,9 @@ def _sl_validate_query_result(query_result: Mapping[str, Any]) -> dict[str, Any] return dict(query_result) -def verify_sbom_lineage_query(query_result: Mapping[str, Any]) -> dict[str, Any]: +def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: try: - checked = _sl_validate_query_result(query_result) + checked = _sl_validate_query_result(query_result, authoritative_lineage) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} @@ -1800,16 +1931,28 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: elif command == "merge": documents = [] for path in args.documents: - raw_bytes = _sl_read_bounded(Path(path)) - try: - payload = json.loads(raw_bytes.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - payload = None + payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) - edges = json.loads(_sl_read_bounded(Path(args.edges)).decode("utf-8")) if getattr(args, "edges", None) else [] - document = build_sbom_lineage(documents, edges=edges) + raw_documents = getattr(args, "raw_documents", None) + raw_inputs = None + if raw_documents is not None: + if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents): + raise SBOMLineageError("raw_documents must contain one raw source per document") + raw_inputs = [] + for raw_path in raw_documents: + _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) + raw_inputs.append(raw_bytes) + if getattr(args, "edges", None): + edges, _ = _sl_load_json_bounded(Path(args.edges)) + if not isinstance(edges, list): + raise SBOMLineageError("lineage edges JSON must be a list") + else: + edges = [] + document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) elif command == "query": - lineage = json.loads(_sl_read_bounded(Path(args.lineage)).decode("utf-8")) + lineage, _ = _sl_load_json_bounded(Path(args.lineage)) + if not isinstance(lineage, Mapping): + raise SBOMLineageError("lineage JSON root must be an object") document = query_sbom_lineage(lineage, args.component, limit=getattr(args, "limit", 32)) else: raise SBOMLineageError("command must be ingest, merge, or query") diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 88c136f0..94eb3708 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -481,3 +481,132 @@ def test_cyclonedx_purl_qualifiers_are_valid_component_ids(): document = perseus.ingest_sbom_document(payload) assert any(component["component_id"] == qualified for component in document["components"]) assert perseus.verify_sbom_document(document)["valid"] is True + + +def test_percent_encoded_secret_surfaces_are_not_projected(): + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["purl"] = "pkg:maven/example/pkg@1?%61pi_key=RAW_PURL" + component["externalReferences"] = [{ + "type": "website", + "url": "https://%75ser:%70w@example.invalid/?%74oken=RAW_URL", + "comment": "%42earer%20RAW_COMMENT", + }] + document = perseus.ingest_sbom_document(payload, source_ref="artifact:%74oken=RAW_SOURCE") + encoded = json.dumps(document, sort_keys=True) + for secret in ("RAW_PURL", "RAW_URL", "RAW_COMMENT", "RAW_SOURCE"): + assert secret not in encoded + + +def test_raw_identifier_fields_reject_non_string_values(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["bom-ref"] = 123 + with pytest.raises(perseus.SBOMLineageError, match="string"): + perseus.ingest_sbom_document(payload) + + +def test_recomputed_projection_digest_without_raw_binding_is_rejected(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:fixture") + forged = json.loads(json.dumps(document)) + forged["components"][0]["name"] = "forged-component" + unsigned = dict(forged) + unsigned.pop("ingestion_digest") + forged["ingestion_digest"] = perseus._sl_ingestion_sha(unsigned) + with pytest.raises(perseus.SBOMLineageError, match="raw|source|bound|digest"): + perseus.build_sbom_lineage([forged]) + + +def test_query_verifier_rejects_self_consistent_path_outside_authoritative_lineage(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + forged = json.loads(json.dumps(result)) + forged["impacted_artifacts"][0]["path"][0]["type"] = "forged_relation" + _reseal(forged) + assert perseus.verify_sbom_lineage_query(forged)["valid"] is False + + +def test_lineage_builder_rejects_node_cap_before_return(): + documents = [] + batches = perseus._SL_MAX_NODES // perseus._SL_MAX_COMPONENTS + 1 + for batch in range(batches): + payload = { + "spdxVersion": "SPDX-2.3", + "SPDXID": f"SPDXRef-DOCUMENT-{batch}", + "packages": [ + { + "SPDXID": f"SPDXRef-Pkg-{batch}-{index}", + "name": f"pkg-{batch}-{index}", + "versionInfo": "1.0", + "supplier": "Organization: Test", + } + for index in range(perseus._SL_MAX_COMPONENTS) + ], + "relationships": [], + } + documents.append(perseus.ingest_sbom_document(payload, source_ref=f"artifact:node-cap-{batch}")) + with pytest.raises(perseus.SBOMLineageError, match="nodes"): + perseus.build_sbom_lineage(documents, edges=[]) + + +def test_cli_json_reads_fail_closed_on_deep_nesting(tmp_path, capsys): + depth = max(perseus._SL_MAX_JSON_DEPTH + 1, 10000) + deep = b"{" + b'"x":{' * depth + b'"leaf":0' + b"}" * (depth + 1) + deep_path = tmp_path / "deep.json" + deep_path.write_bytes(deep) + merge_args = type("Args", (), { + "sbom_command": "merge", "documents": [str(deep_path)], "edges": None, + "output": None, "json": True, + })() + query_args = type("Args", (), { + "sbom_command": "query", "lineage": str(deep_path), "component": "x", + "limit": 32, "output": None, "json": True, + })() + assert perseus.cmd_sbom(merge_args, {}) == 1 + assert perseus.cmd_sbom(query_args, {}) == 1 + assert "nesting" in capsys.readouterr().out + + +def test_duplicate_endpoint_edges_are_canonicalized_across_relationship_types(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") + edges = [ + {"from": "source:stable", "to": "artifact:stable", "type": "generates", "confidence": "high", "coverage": "complete"}, + {"from": "source:stable", "to": "artifact:stable", "type": "derived_from", "confidence": "unknown", "coverage": "unknown", "evidence_refs": ["ledger:strong"]}, + ] + lineage = perseus.build_sbom_lineage([document], edges=edges) + selected = [edge for edge in lineage["edges"] if edge["from"] == "source:stable"] + assert len(selected) == 1 + assert selected[0]["evidence_refs"] == ["ledger:strong"] + + +def test_lineage_builder_rebinds_persisted_documents_to_raw_bytes(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:cross-process") + perseus._SL_INGESTED_PROVENANCE.clear() + lineage = perseus.build_sbom_lineage([document], edges=[], raw_documents=[raw]) + assert perseus.verify_sbom_lineage(lineage)["valid"] is True + + +def test_rebinding_accepts_an_already_sanitized_source_reference(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:token=RAW_SOURCE") + assert document["source_ref"].startswith("sha256:source-ref:") + perseus._SL_INGESTED_PROVENANCE.clear() + lineage = perseus.build_sbom_lineage([document], edges=[], raw_documents=[raw]) + assert perseus.verify_sbom_lineage(lineage)["valid"] is True + + +def test_cli_merge_rebinds_normalized_documents_with_raw_documents(tmp_path, capsys): + raw = _load("spdx-app.json") + raw_path = tmp_path / "raw.json" + normalized_path = tmp_path / "normalized.json" + raw_path.write_bytes(raw) + normalized_path.write_text(json.dumps(perseus.ingest_sbom_document(raw, source_ref="artifact:cross-process")), encoding="utf-8") + perseus._SL_INGESTED_PROVENANCE.clear() + args = type("Args", (), { + "sbom_command": "merge", "documents": [str(normalized_path)], + "raw_documents": [str(raw_path)], "edges": None, "output": None, "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 0 + assert "lineage_digest" in capsys.readouterr().out From 261d2e7507bb6109807ab3c9d753def18cd2545f Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 17:29:07 +0000 Subject: [PATCH 04/40] fix(sbom): close lineage verification bypasses --- perseus.py | 19 ++++++++++--- src/perseus/sbom_lineage.py | 17 +++++++++--- tests/test_sbom_lineage.py | 54 +++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 7 deletions(-) diff --git a/perseus.py b/perseus.py index f0cb36a8..296bdd35 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "8610f04-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "17f53e2-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44542,9 +44542,10 @@ def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" candidates = [value] decoded = value - # Decode a small, bounded number of layers so encoded URL/PURL userinfo - # and query keys cannot evade the credential checks below. - for _ in range(3): + # Decode until stable, bounded by the scalar length. Each successful + # percent-decoding pass must consume at least one encoded triplet, so this + # remains bounded for the 1024-character input limit. + for _ in range(min(len(value) + 1, 1025)): next_decoded = unquote(decoded) if next_decoded == decoded: break @@ -44623,6 +44624,8 @@ def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: def _sl_safe_locator(value: Any, field: str) -> str: """Keep public identifiers; hash credential-bearing scalar values.""" + if not isinstance(value, str): + raise SBOMLineageError(f"{field} must be a string") text = _sl_text(value, field, required=True, limit=1024) if _sl_sensitive(text): return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -46042,10 +46045,15 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: if full_component_ids != set(component_map): raise SBOMLineageError("lineage component nodes are not bound to all documents") checked_edges = [] + edge_endpoints: set[tuple[str, str]] = set() for edge in edges: checked = _sl_validate_relationship(edge, field="lineage.edge") if checked["from"] not in node_ids or checked["to"] not in node_ids: raise SBOMLineageError("lineage edge endpoint is not bound to a node") + endpoint = (checked["from"], checked["to"]) + if endpoint in edge_endpoints: + raise SBOMLineageError("lineage contains duplicate edge endpoints") + edge_endpoints.add(endpoint) checked_edges.append(checked) coverage = lineage.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "truncated"}, set(), "lineage.coverage") @@ -46270,6 +46278,9 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin _sl_id(node_id, "matched_node") if not set(matched_nodes).issubset(authority_node_ids): raise SBOMLineageError("query matched nodes are not bound to the authoritative lineage") + expected_matches = _sl_query_matches(lineage_nodes, query)[:_SL_MAX_QUERY_MATCHES] + if matched_nodes != expected_matches: + raise SBOMLineageError("query matched nodes are inconsistent with the query text") impacted = _sl_list(query_result.get("impacted_artifacts"), "impacted_artifacts") if len(impacted) > _SL_MAX_QUERY_RESULTS: raise SBOMLineageError("impacted_artifacts exceeds its bound") diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 32f11b1f..312f213f 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -106,9 +106,10 @@ def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" candidates = [value] decoded = value - # Decode a small, bounded number of layers so encoded URL/PURL userinfo - # and query keys cannot evade the credential checks below. - for _ in range(3): + # Decode until stable, bounded by the scalar length. Each successful + # percent-decoding pass must consume at least one encoded triplet, so this + # remains bounded for the 1024-character input limit. + for _ in range(min(len(value) + 1, 1025)): next_decoded = unquote(decoded) if next_decoded == decoded: break @@ -187,6 +188,8 @@ def _sl_id(value: Any, field: str, *, fallback: str = "") -> str: def _sl_safe_locator(value: Any, field: str) -> str: """Keep public identifiers; hash credential-bearing scalar values.""" + if not isinstance(value, str): + raise SBOMLineageError(f"{field} must be a string") text = _sl_text(value, field, required=True, limit=1024) if _sl_sensitive(text): return "sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest() @@ -1606,10 +1609,15 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: if full_component_ids != set(component_map): raise SBOMLineageError("lineage component nodes are not bound to all documents") checked_edges = [] + edge_endpoints: set[tuple[str, str]] = set() for edge in edges: checked = _sl_validate_relationship(edge, field="lineage.edge") if checked["from"] not in node_ids or checked["to"] not in node_ids: raise SBOMLineageError("lineage edge endpoint is not bound to a node") + endpoint = (checked["from"], checked["to"]) + if endpoint in edge_endpoints: + raise SBOMLineageError("lineage contains duplicate edge endpoints") + edge_endpoints.add(endpoint) checked_edges.append(checked) coverage = lineage.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "truncated"}, set(), "lineage.coverage") @@ -1834,6 +1842,9 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin _sl_id(node_id, "matched_node") if not set(matched_nodes).issubset(authority_node_ids): raise SBOMLineageError("query matched nodes are not bound to the authoritative lineage") + expected_matches = _sl_query_matches(lineage_nodes, query)[:_SL_MAX_QUERY_MATCHES] + if matched_nodes != expected_matches: + raise SBOMLineageError("query matched nodes are inconsistent with the query text") impacted = _sl_list(query_result.get("impacted_artifacts"), "impacted_artifacts") if len(impacted) > _SL_MAX_QUERY_RESULTS: raise SBOMLineageError("impacted_artifacts exceeds its bound") diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 94eb3708..74ef42de 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -123,6 +123,60 @@ def test_credential_bearing_source_and_references_are_not_persisted(): assert document["source_ref"].startswith("sha256:") +def test_percent_encoded_secrets_are_redacted_from_all_reference_depths(): + payload = json.loads(_load("spdx-app.json")) + payload["packages"][0]["externalRefs"].append( + { + "referenceCategory": "SECURITY", + "referenceType": "website", + "referenceLocator": "https://example.invalid/?%25252561pi_key=RAW_DEEP", + } + ) + document = perseus.ingest_sbom_document(payload, source_ref="artifact:a%25252574oken:RAW_DEEP_SOURCE") + serialized = json.dumps(document, sort_keys=True) + assert "RAW_DEEP" not in serialized + assert "api_key" not in serialized.casefold() + assert document["source_ref"].startswith("sha256:") + + +def test_raw_reference_scalars_must_be_strings(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["purl"] = 123 + with pytest.raises(perseus.SBOMLineageError, match="purl.*string|component_identifier.*string"): + perseus.ingest_sbom_document(payload) + + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [{"type": "website", "url": {"note": "RAW_URL_OBJECT"}}] + with pytest.raises(perseus.SBOMLineageError, match="reference_locator.*string"): + perseus.ingest_sbom_document(payload) + + +def test_query_result_must_bind_query_text_to_matched_nodes(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:query-binding") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + forged = dict(result) + forged["query"] = "term-that-does-not-match" + forged.pop("query_digest") + forged["query_digest"] = perseus._sl_sha(forged) + verification = perseus.verify_sbom_lineage_query(forged, lineage) + assert verification["valid"] is False + + +def test_persisted_lineage_rejects_duplicate_endpoint_edges(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:duplicate-edge") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + forged = json.loads(json.dumps(lineage)) + duplicate = dict(forged["edges"][0]) + duplicate["type"] = "alternate_relation" + forged["edges"].append(duplicate) + unsigned = dict(forged) + unsigned.pop("lineage_digest") + forged["lineage_digest"] = perseus._sl_sha(unsigned) + verification = perseus.verify_sbom_lineage(forged) + assert verification["valid"] is False + + def test_untrusted_normalized_documents_and_conflicting_duplicate_ids_fail_closed(): document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture") tampered = json.loads(json.dumps(document)) From 01eb2100ba375dd5a332da0e679e1947dc4abb19 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 20:14:03 +0000 Subject: [PATCH 05/40] fix(sbom): close independent review blockers --- docs/SBOM.md | 6 +- src/perseus/cli.py | 3 +- src/perseus/sbom_lineage.py | 333 ++++++++++++++++++++++++------ tests/test_sbom_lineage.py | 393 ++++++++++++++++++++++++++++++++++++ 4 files changed, 665 insertions(+), 70 deletions(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index 36a81b1c..54458d8b 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -132,8 +132,10 @@ Example offline commands: ```bash perseus sbom ingest build.spdx.json --output normalized.json -perseus sbom merge normalized.json --edges pipeline-edges.json --output lineage.json -perseus sbom query lineage.json CVE-2021-44228 --json +# Persisted normalized documents must be rebound to their raw source at merge. +perseus sbom merge normalized.json --raw-documents build.spdx.json --edges pipeline-edges.json --output lineage.json +# Persisted lineage must be rebound again at query in a new process. +perseus sbom query lineage.json CVE-2021-44228 --raw-documents build.spdx.json --json ``` The core path requires no cloud service. Deterministic JSON/XML fixtures and diff --git a/src/perseus/cli.py b/src/perseus/cli.py index a79864ac..fbf726c3 100644 --- a/src/perseus/cli.py +++ b/src/perseus/cli.py @@ -162,7 +162,7 @@ def main(): p_sbom_ingest.add_argument("--json", action="store_true", help="Print machine-readable JSON") p_sbom_merge = sbom_sub.add_parser("merge", help="Build a queryable lineage graph from normalized SBOM documents") p_sbom_merge.add_argument("documents", nargs="+", help="SBOM document paths") - p_sbom_merge.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to normalized documents") + p_sbom_merge.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to normalized documents; required for persisted normalized inputs") p_sbom_merge.add_argument("--edges", default=None, help="Optional JSON file of source/build/artifact/deployment edges") p_sbom_merge.add_argument("--output", "-o", default=None, help="Write the lineage graph to a JSON file") p_sbom_merge.add_argument("--json", action="store_true", help="Print machine-readable JSON") @@ -170,6 +170,7 @@ def main(): p_sbom_query.add_argument("lineage", help="Lineage graph JSON path") p_sbom_query.add_argument("component", help="Component name, version, purl, or vulnerability reference") p_sbom_query.add_argument("--limit", type=int, default=32, help="Maximum impacted artifacts") + p_sbom_query.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to lineage documents; required across processes") p_sbom_query.add_argument("--output", "-o", default=None, help="Write the query result to a JSON file") p_sbom_query.add_argument("--json", action="store_true", help="Print machine-readable JSON") diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 312f213f..7da0ab6f 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -10,7 +10,10 @@ import hashlib import json +import math +import os as _sl_os import re +import stat as _sl_stat import xml.etree.ElementTree as _sl_et from urllib.parse import unquote from collections import deque @@ -30,7 +33,9 @@ _SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") -_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer\s+|basic\s+|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*=|credential\s*=)") +_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=)") +_SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw)(?:[/?:=&_.-]|$)") +_SL_PUBLIC_PURL_QUERY_KEYS = frozenset({"classifier", "extension", "type", "repository_url"}) _SL_REFERENCE_TYPES = frozenset({ "advisory", "attestation", "cve", "distribution", "documentation", "license", "purl", "signature", "vex", "vulnerability", "website", "other", @@ -82,7 +87,7 @@ # is caller-recomputable; only a projection produced by raw ingestion (or one # explicitly re-verified against raw bytes) is accepted by lineage builders. _SL_INGESTED_PROVENANCE: dict[str, tuple[str, str]] = {} -_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str]] = {} +_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str, str]] = {} class SBOMLineageError(ValueError): @@ -93,6 +98,19 @@ def _sl_json(value: Any) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) +def _sl_json_object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise SBOMLineageError("duplicate JSON object key") + result[key] = value + return result + + +def _sl_reject_json_constant(value: str) -> None: + raise SBOMLineageError(f"non-finite JSON constant is not allowed: {value}") + + def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() @@ -102,6 +120,29 @@ def _sl_ingestion_sha(unsigned: Mapping[str, Any], raw_bytes: bytes | None = Non return _sl_sha({"domain": "perseus-sbom-ingestion", "raw_sha256": raw_digest, "projection": unsigned}) +def _sl_private_locator(value: str) -> bool: + """Identify private/credential-bearing URI and identifier shapes.""" + decoded = value + for _ in range(min(len(value) + 1, 1025)): + next_decoded = unquote(decoded) + if next_decoded == decoded: + break + decoded = next_decoded + lowered = decoded.casefold() + if "#" in decoded or lowered.startswith("file:") or _SL_PRIVATE_LOCATOR_RE.search(decoded): + return True + authority = decoded.split("://", 1)[1].split("/", 1)[0] if "://" in decoded else "" + if authority and "@" in authority: + return True + if "?" in decoded: + query = decoded.split("?", 1)[1].split("#", 1)[0] + for pair in query.split("&"): + key = pair.split("=", 1)[0].casefold() + if key and key not in _SL_PUBLIC_PURL_QUERY_KEYS: + return True + return False + + def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" candidates = [value] @@ -116,11 +157,10 @@ def _sl_sensitive(value: str) -> bool: candidates.append(next_decoded) decoded = next_decoded for candidate in candidates: - authority = candidate.split("://", 1)[1].split("/", 1)[0] if "://" in candidate else "" normalized = re.sub(r"[^a-z0-9]+", "_", candidate.casefold()) if ( _SL_SENSITIVE_REFERENCE_RE.search(candidate) - or (authority and "@" in authority) + or _sl_private_locator(candidate) or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) ): return True @@ -161,7 +201,7 @@ def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512 raise SBOMLineageError(f"{field} is required") return "" if not isinstance(value, str): - value = str(value) + raise SBOMLineageError(f"{field} must be a string") text = re.sub(r"[\x00-\x1f\x7f]", " ", value).strip() if required and not text: raise SBOMLineageError(f"{field} is required") @@ -197,8 +237,10 @@ def _sl_safe_locator(value: Any, field: str) -> str: def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: - if value in (None, ""): + if value == "": return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" + if not isinstance(value, str): + raise SBOMLineageError("source_ref must be a string") text = _sl_text(value, "source_ref", required=True, limit=512) raw_digest = hashlib.sha256(raw_bytes).hexdigest() if text.startswith("sha256:source-ref:"): @@ -451,10 +493,17 @@ def _sl_component( component_type: Any = None, licenses: Any = None, truncated: list[str] | None = None, + component_id_map: dict[str, str] | None = None, ) -> dict[str, Any]: normalized_name = _sl_safe_text(name, "component_name", required=True, limit=256) normalized_version = _sl_safe_text(version, "component_version", limit=128) normalized_id = _sl_id(component_id, "component ID") + if isinstance(component_id, str) and component_id_map is not None: + component_id_map[component_id] = normalized_id + if _sl_node_kind(normalized_id) != "component": + normalized_id = "component:sha256:" + hashlib.sha256(normalized_id.encode("utf-8")).hexdigest() + if isinstance(component_id, str) and component_id_map is not None: + component_id_map[component_id] = normalized_id ids = {normalized_id} component_truncated: list[str] = list(truncated or []) if identifiers is not None and not isinstance(identifiers, (list, tuple, set)): @@ -466,6 +515,9 @@ def _sl_component( text = _sl_safe_locator(identifier, "component_identifier") if text: ids.add(text) + if component_id_map is not None: + for identifier in ids: + component_id_map.setdefault(identifier, normalized_id) safe_references = [] if references is not None and not isinstance(references, (list, tuple)): raise SBOMLineageError("component references must be a list") @@ -514,12 +566,17 @@ def _sl_component( "identifiers": sorted(ids), "references": sorted(safe_references, key=lambda item: (item.get("type", ""), item.get("locator", ""))), "licenses": sorted(set(license_values)), - "component_type": _sl_text(component_type, "component_type", limit=64) or "unknown", + "component_type": _sl_safe_text(component_type, "component_type", limit=128) or "unknown", "coverage": {"state": coverage_state, "unknown": sorted(set(unknown)), "truncated": sorted(set(component_truncated))}, } -def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None, component_id_map: Mapping[str, str] | None = None) -> dict[str, Any]: + if component_id_map is not None: + if isinstance(source, str): + source = component_id_map.get(source, source) + if isinstance(target, str): + target = component_id_map.get(target, target) source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") @@ -546,7 +603,7 @@ def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confid return result -def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] references, identifiers = _sl_spdx_references(raw.get("externalRefs"), truncated=component_truncated) if truncated is not None: @@ -561,10 +618,11 @@ def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = component_type="package", licenses=raw.get("licenseConcluded"), truncated=component_truncated, + component_id_map=component_id_map, ) -def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] references, identifiers = _sl_cdx_references(raw.get("externalReferences"), truncated=component_truncated) if "purl" in raw and raw.get("purl") is not None: @@ -572,7 +630,7 @@ def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = N references.extend(_sl_properties(raw.get("properties"), truncated=component_truncated)) if truncated is not None: truncated.extend(item for item in component_truncated if item not in truncated) - return _sl_component( + component = _sl_component( component_id=raw.get("bom-ref"), name=raw.get("name"), version=raw.get("version"), @@ -582,7 +640,11 @@ def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = N component_type=raw.get("type"), licenses=raw.get("licenses"), truncated=component_truncated, + component_id_map=component_id_map, ) + if component_id_map is not None and isinstance(raw.get("purl"), str): + component_id_map.setdefault(raw["purl"], component["component_id"]) + return component def _sl_validate_spdx_version(value: Any) -> str: @@ -600,6 +662,13 @@ def _sl_validate_cdx_version(value: Any) -> str: return version_text +def _sl_spdx_document_id(value: Any) -> str: + document_id = _sl_id(value, "SPDXID") + if _sl_node_kind(document_id) != "document": + raise SBOMLineageError("SPDXID must use the document namespace") + return document_id + + def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") @@ -607,12 +676,15 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu raise SBOMLineageError("normalized SBOM components exceed their global bound") if len(relationships) > _SL_MAX_RELATIONSHIPS: raise SBOMLineageError("normalized SBOM relationships exceed their global bound") + safe_supplier = _sl_safe_text(supplier, "supplier", limit=512) component_ids: set[str] = set() for item in components: component_id = item.get("component_id") if isinstance(item, Mapping) else None if not isinstance(component_id, str) or component_id in component_ids: raise SBOMLineageError("duplicate or missing component ID") component_ids.add(component_id) + if fmt == "SPDX" and document_id in component_ids: + raise SBOMLineageError("SPDX document ID collides with a component ID") known_ids = set(component_ids) if fmt == "SPDX": known_ids.add(document_id) @@ -627,7 +699,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("document_name") if not created_at: unknown.append("created_at") - if not supplier: + if not safe_supplier: unknown.append("supplier") if dangling: unknown.append("dangling_relationships") @@ -637,7 +709,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("component_metadata") if not components: coverage_state = "unknown" - elif dangling or truncation or not document_name or not created_at or not supplier or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: + elif dangling or truncation or not document_name or not created_at or not safe_supplier or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: coverage_state = "partial" else: coverage_state = "complete" @@ -650,7 +722,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "document_sha256": hashlib.sha256(raw_bytes).hexdigest(), "source_ref": source_ref, "created_at": _sl_safe_text(created_at, "created_at", limit=128) or None, - "supplier": supplier or None, + "supplier": safe_supplier or None, "components": sorted(components, key=lambda item: (item["component_id"] == metadata_component_id, item["component_id"])), "relationships": sorted(relationships, key=lambda item: (item["from"], item["to"], item["type"])), "coverage": { @@ -672,7 +744,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: str) -> dict[str, Any]: version = _sl_validate_spdx_version(value.get("spdxVersion")) - document_id = _sl_id(value.get("SPDXID"), "SPDXID") + document_id = _sl_spdx_document_id(value.get("SPDXID")) creation = value.get("creationInfo", {}) if creation is None: creation = {} @@ -689,18 +761,19 @@ def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: truncated.append("packages") if len(raw_relationships) > _SL_MAX_RELATIONSHIPS: truncated.append("relationships") + component_id_map: dict[str, str] = {} components = [] for item in packages[:_SL_MAX_COMPONENTS]: if not isinstance(item, Mapping): raise SBOMLineageError("packages must contain objects") - components.append(_sl_spdx_component(item, truncated=truncated)) + components.append(_sl_spdx_component(item, truncated=truncated, component_id_map=component_id_map)) relationships = [] for raw in raw_relationships[:_SL_MAX_RELATIONSHIPS]: if not isinstance(raw, Mapping): raise SBOMLineageError("relationships must contain objects") relationships.append(_sl_relationship( raw.get("spdxElementId"), raw.get("relatedSpdxElement"), - raw.get("relationshipType", "related"), truncated=truncated, + raw.get("relationshipType", "related"), truncated=truncated, component_id_map=component_id_map, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -738,8 +811,9 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s raise SBOMLineageError("components must contain objects") raw_components.append(item) components: list[dict[str, Any]] = [] + component_id_map: dict[str, str] = {} for raw in raw_components: - components.append(_sl_cdx_component(raw, truncated=truncated)) + components.append(_sl_cdx_component(raw, truncated=truncated, component_id_map=component_id_map)) relationships = [] for raw in raw_dependency_list[:_SL_MAX_RELATIONSHIPS]: if not isinstance(raw, Mapping): @@ -756,12 +830,15 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s if len(targets) > allowed_targets: truncated.append("dependency_edges") for target in targets[:allowed_targets]: - relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated)) + relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated, component_id_map=component_id_map)) creators = metadata.get("authors", []) if not isinstance(creators, list): raise SBOMLineageError("metadata.authors must be a list") supplier = _sl_supplier(creators[0] if creators else (metadata_component or {})) - document_id = _sl_id(value.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + if "serialNumber" in value: + document_id = _sl_id(value.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_text(metadata.get("name"), "document_name"), created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, @@ -777,7 +854,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") if not raw_document_id: raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") - document_id = _sl_id(raw_document_id, "SPDXID") + document_id = _sl_spdx_document_id(raw_document_id) creation = next(iter(_sl_descendants(document, "creationInfo")), None) created_at = _sl_xml_value(creation, "created") creator = _sl_xml_value(creation, "creator") @@ -788,6 +865,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, truncated.append("packages") if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: truncated.append("relationships") + component_id_map: dict[str, str] = {} components = [] for raw in packages[:_SL_MAX_COMPONENTS]: component_truncated: list[str] = [] @@ -818,7 +896,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, references=refs, component_type="package", licenses=_sl_xml_value(raw, "licenseConcluded"), - truncated=component_truncated, + truncated=component_truncated, component_id_map=component_id_map, )) relationships = [] for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: @@ -826,7 +904,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, _sl_xml_value(raw, "spdxElementId"), _sl_xml_value(raw, "relatedSpdxElement"), _sl_xml_value(raw, "relationshipType", default="related"), - truncated=truncated, + truncated=truncated, component_id_map=component_id_map, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -837,7 +915,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) - document_id = _sl_id(_sl_xml_text(root, "SPDXID"), "SPDXID") + document_id = _sl_spdx_document_id(_sl_xml_text(root, "SPDXID")) creation = _sl_child(root, "creationInfo") created_at = _sl_xml_text(creation, "created") if creation is not None else "" creator = _sl_xml_text(creation, "creator") if creation is not None else "" @@ -848,6 +926,7 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] truncated.append("packages") if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: truncated.append("relationships") + component_id_map: dict[str, str] = {} components = [] for raw in packages[:_SL_MAX_COMPONENTS]: component_truncated: list[str] = [] @@ -871,14 +950,14 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] version=_sl_xml_text(raw, "versionInfo"), supplier=_sl_xml_text(raw, "supplier"), identifiers=identifiers, references=refs, component_type="package", licenses=_sl_xml_text(raw, "licenseConcluded"), - truncated=component_truncated, + truncated=component_truncated, component_id_map=component_id_map, )) relationships = [] for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: relationships.append(_sl_relationship( _sl_xml_text(raw, "spdxElementId"), _sl_xml_text(raw, "relatedSpdxElement"), _sl_xml_text(raw, "relationshipType", default="related"), - truncated=truncated, + truncated=truncated, component_id_map=component_id_map, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -887,7 +966,7 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] ) -def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] refs = [] identifiers = [] @@ -917,12 +996,15 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None) -> di licenses.append(_sl_xml_text(item, "id") or _sl_xml_text(item, "name")) if truncated is not None: truncated.extend(item for item in component_truncated if item not in truncated) - return _sl_component( + component = _sl_component( component_id=raw.attrib.get("bom-ref"), name=_sl_xml_text(raw, "name"), version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_child(raw, "supplier"), "name"), identifiers=identifiers, references=refs, component_type=raw.attrib.get("type"), licenses=licenses, - truncated=component_truncated, + truncated=component_truncated, component_id_map=component_id_map, ) + if component_id_map is not None and purl: + component_id_map.setdefault(purl, component["component_id"]) + return component def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: @@ -944,9 +1026,10 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if len(component_nodes) > component_budget: truncated.append("components") raw_components.extend(component_nodes[:component_budget]) + component_id_map: dict[str, str] = {} components = [] for raw in raw_components: - components.append(_sl_cdx_xml_component(raw, truncated=truncated)) + components.append(_sl_cdx_xml_component(raw, truncated=truncated, component_id_map=component_id_map)) relationships = [] dependencies_node = _sl_child(root, "dependencies") if dependencies_node is not None: @@ -966,11 +1049,14 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if len(children) > allowed_targets: truncated.append("dependency_edges") for child in children[:allowed_targets]: - relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated)) + relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated, component_id_map=component_id_map)) timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" authors = _sl_child(metadata, "authors") if metadata is not None else None author = _sl_xml_text(_sl_child(authors, "author"), "name") if authors is not None else "" - document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + if "serialNumber" in root.attrib: + document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, @@ -980,20 +1066,42 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_read_bounded(path: Path) -> bytes: - """Read a file only after checking its size, including race re-check.""" + """Read a regular file through a no-follow descriptor with a hard cap.""" try: - size = path.stat().st_size + initial = _sl_os.lstat(path) + if not _sl_stat.S_ISREG(initial.st_mode): + raise SBOMLineageError("SBOM input must be a regular file") + flags = _sl_os.O_RDONLY | getattr(_sl_os, "O_CLOEXEC", 0) | getattr(_sl_os, "O_NOFOLLOW", 0) | getattr(_sl_os, "O_NONBLOCK", 0) + fd = _sl_os.open(path, flags) + except SBOMLineageError: + raise except OSError as exc: - raise SBOMLineageError(f"could not stat SBOM: {exc}") from exc - if size > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + raise SBOMLineageError(f"could not open SBOM safely: {exc}") from exc try: - raw_bytes = path.read_bytes() + opened = _sl_os.fstat(fd) + if not _sl_stat.S_ISREG(opened.st_mode): + raise SBOMLineageError("SBOM input must be a regular file") + initial_identity = (getattr(initial, "st_dev", None), getattr(initial, "st_ino", None)) + opened_identity = (getattr(opened, "st_dev", None), getattr(opened, "st_ino", None)) + if None not in initial_identity + opened_identity and opened_identity != initial_identity: + raise SBOMLineageError("SBOM input changed during safe open") + if opened.st_size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + data = bytearray() + while len(data) <= _SL_MAX_INPUT_BYTES: + chunk = _sl_os.read(fd, min(64 * 1024, _SL_MAX_INPUT_BYTES + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + if len(data) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return bytes(data) + except SBOMLineageError: + raise except OSError as exc: - raise SBOMLineageError(f"could not read SBOM: {exc}") from exc - if len(raw_bytes) > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") - return raw_bytes + raise SBOMLineageError(f"could not read SBOM safely: {exc}") from exc + finally: + _sl_os.close(fd) def _sl_validate_json_depth(raw_bytes: bytes) -> None: @@ -1019,12 +1127,30 @@ def _sl_validate_json_depth(raw_bytes: bytes) -> None: depth = max(0, depth - 1) +def _sl_validate_json_values(value: Any) -> None: + stack = [value] + while stack: + current = stack.pop() + if isinstance(current, float) and not math.isfinite(current): + raise SBOMLineageError("non-finite JSON number is not allowed") + if isinstance(current, Mapping): + stack.extend(current.values()) + elif isinstance(current, list): + stack.extend(current) + + def _sl_load_json_bounded(source: Path | bytes, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: """Load JSON through the shared byte and nesting bounds.""" raw_bytes = _sl_read_bounded(source) if isinstance(source, Path) else bytes(source) _sl_validate_json_depth(raw_bytes) try: - return json.loads(raw_bytes.decode("utf-8")), raw_bytes + value = json.loads( + raw_bytes.decode("utf-8"), + object_pairs_hook=_sl_json_object_pairs, + parse_constant=_sl_reject_json_constant, + ) + _sl_validate_json_values(value) + return value, raw_bytes except RecursionError as exc: raise SBOMLineageError("SBOM JSON nesting is too deep") from exc except (UnicodeDecodeError, json.JSONDecodeError) as exc: @@ -1090,7 +1216,12 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raise SBOMLineageError("SBOM input is empty") try: _sl_validate_json_depth(raw_bytes) - value = json.loads(raw_bytes.decode("utf-8")) + value = json.loads( + raw_bytes.decode("utf-8"), + object_pairs_hook=_sl_json_object_pairs, + parse_constant=_sl_reject_json_constant, + ) + _sl_validate_json_values(value) if not isinstance(value, Mapping): raise SBOMLineageError("SBOM JSON root must be an object") return dict(value), raw_bytes @@ -1127,7 +1258,7 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An def _sl_node_kind(node_id: str) -> str: - if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("document:"): + if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("SPDXRef-DOCUMENT-") or node_id.startswith("document:"): return "document" if node_id.startswith("artifact:"): return "artifact" @@ -1227,7 +1358,7 @@ def _sl_validate_component(component: Any) -> dict[str, Any]: raise SBOMLineageError("component.references exceeds its bound") checked_references = [_sl_validate_reference(reference) for reference in references] licenses = _sl_string_list(component.get("licenses"), "component.licenses", maximum=_SL_MAX_LICENSES) - component_type = _sl_strict_text(component.get("component_type"), "component.component_type", limit=64) + component_type = _sl_strict_text(component.get("component_type"), "component.component_type", limit=128) assert component_type is not None coverage = _sl_validate_component_coverage( component.get("coverage"), has_version=version is not None, has_supplier=supplier is not None, @@ -1344,6 +1475,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non document_id_text = _sl_strict_text(document.get("document_id"), "document_id", limit=256) assert document_id_text is not None document_id = _sl_id(document_id_text, "document_id") + if fmt == "SPDX" and _sl_node_kind(document_id) != "document": + raise SBOMLineageError("SPDX document ID must use the document namespace") for field, limit in (("document_name", 256), ("created_at", 128), ("supplier", 512)): _sl_strict_text(document.get(field), field, allow_none=True, limit=limit) document_sha = document.get("document_sha256") @@ -1364,6 +1497,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non raise SBOMLineageError("normalized SBOM contains duplicate component IDs") seen.add(checked["component_id"]) checked_components.append(checked) + if fmt == "SPDX" and document_id in seen: + raise SBOMLineageError("SPDX document ID collides with a component ID") checked_relationships = [_sl_validate_relationship(item) for item in relationships] coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") @@ -1521,6 +1656,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any } body["lineage_digest"] = _sl_sha(body) _SL_LINEAGE_PROVENANCE[body["lineage_digest"]] = ( + _sl_json(body), _sl_json(body["nodes"]), _sl_json(body["edges"]), ) @@ -1575,7 +1711,7 @@ def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[ return node_id -def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: +def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None = None) -> dict[str, Any]: required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: raise SBOMLineageError("unsupported software-lineage schema") @@ -1584,12 +1720,22 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: unsigned.pop("lineage_digest", None) if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: raise SBOMLineageError("lineage digest mismatch") + trusted = _SL_LINEAGE_PROVENANCE.get(supplied) + if raw_documents is None and (trusted is None or trusted[0] != _sl_json(dict(lineage))): + raise SBOMLineageError("lineage is not bound to a trusted builder receipt or raw documents") documents = _sl_list(lineage.get("documents"), "lineage.documents") nodes = _sl_list(lineage.get("nodes"), "lineage.nodes") edges = _sl_list(lineage.get("edges"), "lineage.edges") + if not documents: + raise SBOMLineageError("lineage must contain at least one document") if len(documents) > _SL_MAX_DOCUMENTS or len(edges) > _SL_MAX_EDGES or len(nodes) > _SL_MAX_NODES: raise SBOMLineageError("lineage collection exceeds its bound") - checked_documents = [_sl_validate_document(document) for document in documents] + if raw_documents is None: + checked_documents = [_sl_validate_document(document) for document in documents] + else: + if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: + raise SBOMLineageError("raw_documents must match lineage documents within its bound") + checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} for document in checked_documents: for component in document["components"]: @@ -1639,9 +1785,9 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: return dict(lineage) -def verify_sbom_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: +def verify_sbom_lineage(lineage: Mapping[str, Any], raw_documents: Any | None = None) -> dict[str, Any]: try: - loaded = _sl_loaded_lineage(lineage) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -1682,9 +1828,9 @@ def _sl_path_confidence(path: list[Mapping[str, Any]]) -> str: return "high" -def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32) -> dict[str, Any]: +def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32, raw_documents: Any | None = None) -> dict[str, Any]: """Find impacted artifact nodes and return every traversed evidence edge.""" - loaded = _sl_loaded_lineage(lineage) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) limit = _sl_limit(limit, "limit") query_text = _sl_strict_text(query, "query", limit=512) assert query_text is not None @@ -1777,6 +1923,7 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 "lineage_nodes": loaded["nodes"], "lineage_edges": loaded["edges"], "query": query_text, + "limit": limit, "matched_nodes": matches, "impacted_artifacts": impacted, "status": status, @@ -1787,8 +1934,8 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 return unsigned -def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: - required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} +def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: + required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") supplied = query_result.get("query_digest") @@ -1805,12 +1952,21 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query authoritative lineage exceeds its bound") authority = _SL_LINEAGE_PROVENANCE.get(lineage_digest) if authoritative_lineage is not None: - loaded_authority = _sl_loaded_lineage(authoritative_lineage) + loaded_authority = _sl_loaded_lineage(authoritative_lineage, raw_documents=raw_documents) if lineage_digest != loaded_authority["lineage_digest"]: raise SBOMLineageError("query result lineage digest does not match the authoritative lineage") - authority = (_sl_json(loaded_authority["nodes"]), _sl_json(loaded_authority["edges"])) - if authority is None or authority != (_sl_json(lineage_nodes), _sl_json(lineage_edges)): + authority = ( + _sl_json(loaded_authority), + _sl_json(loaded_authority["nodes"]), + _sl_json(loaded_authority["edges"]), + ) + if authority is None or authority[1:] != (_sl_json(lineage_nodes), _sl_json(lineage_edges)): raise SBOMLineageError("query result is not bound to the authoritative lineage digest and nodes/edges") + if authoritative_lineage is None: + try: + authoritative_lineage = json.loads(authority[0]) + except (TypeError, json.JSONDecodeError) as exc: + raise SBOMLineageError("authoritative lineage receipt is malformed") from exc authority_node_ids: set[str] = set() for node in lineage_nodes: if not isinstance(node, Mapping): @@ -1837,6 +1993,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin authority_edges.append(checked_edge) query = _sl_strict_text(query_result.get("query"), "query", limit=512) assert query is not None + limit = _sl_limit(query_result.get("limit"), "query.limit") matched_nodes = _sl_string_list(query_result.get("matched_nodes"), "matched_nodes", maximum=_SL_MAX_QUERY_MATCHES) for node_id in matched_nodes: _sl_id(node_id, "matched_node") @@ -1921,17 +2078,46 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query claims are inconsistent with coverage") if status != expected_status: raise SBOMLineageError("query status is inconsistent with coverage") + assert authoritative_lineage is not None + expected_result = query_sbom_lineage( + authoritative_lineage, query, limit=limit, raw_documents=raw_documents, + ) + for field in ( + "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", + "impacted_artifacts", "status", "coverage", "claims", "query_digest", + ): + if query_result.get(field) != expected_result.get(field): + raise SBOMLineageError(f"query field {field} is inconsistent with authoritative recomputation") return dict(query_result) -def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: +def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: try: - checked = _sl_validate_query_result(query_result, authoritative_lineage) + checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} +def _sl_cli_bounded_paths(value: Any, field: str, *, required: bool = False) -> list[str] | None: + if value is None: + if required: + raise SBOMLineageError(f"{field} is required") + return None + if not isinstance(value, (list, tuple)): + raise SBOMLineageError(f"{field} must be a list") + if not value: + raise SBOMLineageError(f"{field} must not be empty") + if len(value) > _SL_MAX_DOCUMENTS: + raise SBOMLineageError(f"{field} exceeds {_SL_MAX_DOCUMENTS} documents") + result: list[str] = [] + for item in value: + if not isinstance(item, str) or not item: + raise SBOMLineageError(f"{field} paths must be strings") + result.append(item) + return result + + def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: """CLI adapter for local SBOM normalization, merge, and query operations.""" try: @@ -1940,17 +2126,18 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: if command == "ingest": document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) elif command == "merge": + document_paths = _sl_cli_bounded_paths(getattr(args, "documents", None), "documents", required=True) + raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") + if raw_document_paths is not None and len(raw_document_paths) != len(document_paths or []): + raise SBOMLineageError("raw_documents must contain one raw source per document") documents = [] - for path in args.documents: + for path in document_paths or []: payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) - raw_documents = getattr(args, "raw_documents", None) raw_inputs = None - if raw_documents is not None: - if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents): - raise SBOMLineageError("raw_documents must contain one raw source per document") + if raw_document_paths is not None: raw_inputs = [] - for raw_path in raw_documents: + for raw_path in raw_document_paths: _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) raw_inputs.append(raw_bytes) if getattr(args, "edges", None): @@ -1961,10 +2148,19 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: edges = [] document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) elif command == "query": + raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage, _ = _sl_load_json_bounded(Path(args.lineage)) if not isinstance(lineage, Mapping): raise SBOMLineageError("lineage JSON root must be an object") - document = query_sbom_lineage(lineage, args.component, limit=getattr(args, "limit", 32)) + raw_inputs = None + if raw_document_paths is not None: + lineage_documents = lineage.get("documents") + if not isinstance(lineage_documents, list) or len(raw_document_paths) != len(lineage_documents): + raise SBOMLineageError("raw_documents must contain one raw source per lineage document") + raw_inputs = [_sl_load_json_bounded(Path(path), allow_non_json=True)[1] for path in raw_document_paths] + document = query_sbom_lineage( + lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, + ) else: raise SBOMLineageError("command must be ingest, merge, or query") serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" @@ -1977,5 +2173,8 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: print(f"sbom {command} -> {output}\n{digest_key}: {document[digest_key]}") return 0 except (OSError, TypeError, ValueError, SBOMLineageError) as exc: - print(f"sbom: {exc}") + if getattr(args, "json", False): + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": str(exc), "valid": False}, sort_keys=True)) + else: + print(f"sbom: {exc}") return 1 diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 74ef42de..13641f91 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -3,6 +3,9 @@ import json import hashlib +import os +import subprocess +import sys from pathlib import Path import pytest @@ -664,3 +667,393 @@ def test_cli_merge_rebinds_normalized_documents_with_raw_documents(tmp_path, cap })() assert perseus.cmd_sbom(args, {}) == 0 assert "lineage_digest" in capsys.readouterr().out + + +def test_cyclonedx_component_type_is_sanitized_before_persistence(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["type"] = "Bearer RAW_COMPONENT_TYPE_SECRET" + payload["metadata"]["component"]["type"] = "Bearer RAW_METADATA_TYPE_SECRET" + document = perseus.ingest_sbom_document(payload, source_ref="artifact:component-type") + serialized = json.dumps(document, sort_keys=True) + assert "RAW_COMPONENT_TYPE_SECRET" not in serialized + assert "RAW_METADATA_TYPE_SECRET" not in serialized + assert document["components"][0]["component_type"].startswith("sha256:") + assert perseus.verify_sbom_document(document)["valid"] is True + + +def test_raw_semantic_scalars_and_spdx_relationship_type_reject_non_strings(): + for source_ref in (123, None, {"raw": "source_ref"}): + with pytest.raises(perseus.SBOMLineageError, match="source_ref.*string"): + perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref=source_ref) + + for field, value in (("name", 123), ("version", 456), ("supplier", {"name": 789})): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0][field] = value + with pytest.raises(perseus.SBOMLineageError, match="string"): + perseus.ingest_sbom_document(payload) + + payload = json.loads(_load("spdx-app.json")) + payload["relationships"][0]["relationshipType"] = 17 + with pytest.raises(perseus.SBOMLineageError, match="string"): + perseus.ingest_sbom_document(payload) + + +def test_component_and_reference_identifier_fields_reject_non_strings(): + cases = [] + + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["bom-ref"] = 123 + cases.append(payload) + + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["purl"] = 123 + cases.append(payload) + + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [{"type": "website", "url": 123}] + cases.append(payload) + + payload = json.loads(_load("cyclonedx-app.json")) + payload["dependencies"][0]["ref"] = 123 + cases.append(payload) + + payload = json.loads(_load("spdx-app.json")) + payload["packages"][0]["externalRefs"][0]["referenceLocator"] = 123 + cases.append(payload) + + payload = json.loads(_load("spdx-app.json")) + payload["relationships"][0]["spdxElementId"] = 123 + cases.append(payload) + + for invalid in cases: + with pytest.raises(perseus.SBOMLineageError, match="string"): + perseus.ingest_sbom_document(invalid) + + +def test_more_than_three_percent_encoded_credentials_never_survive_serialization(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [{ + "type": "website", + "url": "https://example.invalid/?%25252525252574oken=RAW_DEEP_COMPONENT_SECRET", + }] + document = perseus.ingest_sbom_document( + payload, + source_ref="artifact:%25252525252574oken=RAW_DEEP_SOURCE_SECRET", + ) + serialized = json.dumps(document, sort_keys=True) + assert "RAW_DEEP_COMPONENT_SECRET" not in serialized + assert "RAW_DEEP_SOURCE_SECRET" not in serialized + + +def test_query_verifier_rejects_resealed_removed_impacted_artifact(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:query-completeness") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges() + [ + {"from": "SPDXRef-App", "to": "build:perseus-002", "type": "built_into", "confidence": "high", "coverage": "complete"}, + {"from": "build:perseus-002", "to": "artifact:perseus-image@2.0.0", "type": "generates", "confidence": "high", "coverage": "complete"}, + ]) + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + assert len(result["impacted_artifacts"]) == 2 + forged = json.loads(json.dumps(result)) + forged["impacted_artifacts"].pop() + _reseal(forged) + assert perseus.verify_sbom_lineage_query(forged, lineage)["valid"] is False + + +def test_query_verifier_rejects_forged_complete_coverage_from_partial_lineage(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:query-coverage") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + assert result["coverage"]["state"] == "partial" + forged = json.loads(json.dumps(result)) + forged["coverage"]["state"] = "complete" + forged["coverage"]["unknown"] = [] + forged["status"] = "complete" + forged["claims"]["impact_status"] = "established" + forged["coverage"]["truncated"] = [] + _reseal(forged) + assert perseus.verify_sbom_lineage_query(forged, lineage)["valid"] is False + + +def test_query_verifier_requires_authoritative_matched_node_truncation(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:query-match-cap") + edges = [ + {"from": f"source:needle-{index}", "to": "artifact:query-target", "type": "generates", "confidence": "high", "coverage": "complete"} + for index in range(perseus._SL_MAX_QUERY_MATCHES + 1) + ] + lineage = perseus.build_sbom_lineage([document], edges=edges) + result = perseus.query_sbom_lineage(lineage, "needle") + assert "matched_nodes" in result["coverage"]["truncated"] + forged = json.loads(json.dumps(result)) + forged["coverage"]["truncated"].remove("matched_nodes") + _reseal(forged) + assert perseus.verify_sbom_lineage_query(forged, lineage)["valid"] is False + + +def test_query_verifier_rejects_backtracking_and_cyclic_paths(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:query-simple-path") + lineage = perseus.build_sbom_lineage([document], edges=[ + {"from": "SPDXRef-Log4j", "to": "build:cycle", "type": "built_into", "confidence": "high", "coverage": "complete"}, + {"from": "build:cycle", "to": "artifact:cycle", "type": "generates", "confidence": "high", "coverage": "complete"}, + {"from": "SPDXRef-Log4j", "to": "artifact:cycle", "type": "direct", "confidence": "high", "coverage": "complete"}, + ]) + result = perseus.query_sbom_lineage(lineage, "log4j-core") + forged = json.loads(json.dumps(result)) + forged["impacted_artifacts"][0]["path"] = [ + forged["impacted_artifacts"][0]["path"][0], + {"from": "build:cycle", "to": "SPDXRef-Log4j", "type": "built_into", "confidence": "high", "coverage": "complete"}, + {"from": "SPDXRef-Log4j", "to": "artifact:cycle", "type": "direct", "confidence": "high", "coverage": "complete"}, + ] + forged["impacted_artifacts"][0]["coverage"] = "complete" + forged["impacted_artifacts"][0]["confidence"] = "high" + forged["impacted_artifacts"][0]["evidence_refs"] = [] + _reseal(forged) + assert perseus.verify_sbom_lineage_query(forged, lineage)["valid"] is False + + +def test_query_verifier_rejects_a_noncanonical_authoritative_traversal(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:query-canonical-path") + lineage = perseus.build_sbom_lineage([document], edges=[ + {"from": "SPDXRef-Log4j", "to": "build:detour", "type": "built_into", "confidence": "high", "coverage": "complete"}, + {"from": "build:detour", "to": "artifact:canonical", "type": "generates", "confidence": "high", "coverage": "complete"}, + {"from": "SPDXRef-Log4j", "to": "artifact:canonical", "type": "direct", "confidence": "high", "coverage": "complete"}, + ]) + result = perseus.query_sbom_lineage(lineage, "log4j-core") + assert [edge["type"] for edge in result["impacted_artifacts"][0]["path"]] == ["direct"] + forged = json.loads(json.dumps(result)) + forged["impacted_artifacts"][0]["path"] = [ + {"from": "SPDXRef-Log4j", "to": "build:detour", "type": "built_into", "confidence": "high", "coverage": "complete"}, + {"from": "build:detour", "to": "artifact:canonical", "type": "generates", "confidence": "high", "coverage": "complete"}, + ] + _reseal(forged) + assert perseus.verify_sbom_lineage_query(forged, lineage)["valid"] is False + + +def test_verify_sbom_lineage_rejects_resealed_empty_lineage(): + empty = { + "schema_version": "perseus-software-lineage/v1", + "documents": [], + "nodes": [], + "edges": [], + "coverage": {"state": "complete", "unknown": [], "truncated": []}, + } + _reseal(empty) + assert perseus.verify_sbom_lineage(empty)["valid"] is False + + +def test_cli_merge_rejects_more_than_max_documents_before_ingestion(tmp_path, monkeypatch, capsys): + paths = [] + raw = _load("spdx-app.json") + for index in range(perseus._SL_MAX_DOCUMENTS + 1): + path = tmp_path / f"document-{index}.json" + path.write_bytes(raw) + paths.append(str(path)) + calls = [] + original_ingest = perseus.ingest_sbom_document + + def counted_ingest(*args, **kwargs): + calls.append(args[0] if args else None) + return original_ingest(*args, **kwargs) + + monkeypatch.setattr(perseus, "ingest_sbom_document", counted_ingest) + args = type("Args", (), { + "sbom_command": "merge", "documents": paths, "raw_documents": None, + "edges": None, "output": None, "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 1 + assert calls == [] + failure = json.loads(capsys.readouterr().out) + assert failure["valid"] is False + assert "documents" in failure["error"] + + +def test_documented_cli_workflow_rebinds_raw_sources_across_processes(tmp_path): + raw_path = tmp_path / "raw.json" + normalized_path = tmp_path / "normalized.json" + lineage_path = tmp_path / "lineage.json" + query_path = tmp_path / "query.json" + raw_path.write_bytes(_load("spdx-app.json")) + cli = [sys.executable, str(ROOT / "perseus.py"), "sbom"] + + ingest = subprocess.run( + cli + ["ingest", str(raw_path), "--source-ref", "artifact:cross-process", "--output", str(normalized_path), "--json"], + capture_output=True, text=True, check=False, + ) + assert ingest.returncode == 0, ingest.stdout + ingest.stderr + + merge = subprocess.run( + cli + ["merge", str(normalized_path), "--raw-documents", str(raw_path), "--output", str(lineage_path), "--json"], + capture_output=True, text=True, check=False, + ) + assert merge.returncode == 0, merge.stdout + merge.stderr + + without_raw = subprocess.run( + cli + ["query", str(lineage_path), "CVE-2021-44228", "--json"], + capture_output=True, text=True, check=False, + ) + assert without_raw.returncode == 1 + assert json.loads(without_raw.stdout)["valid"] is False + + query = subprocess.run( + cli + ["query", str(lineage_path), "CVE-2021-44228", "--raw-documents", str(raw_path), "--output", str(query_path), "--json"], + capture_output=True, text=True, check=False, + ) + assert query.returncode == 0, query.stdout + query.stderr + assert json.loads(query_path.read_text(encoding="utf-8"))["query"] == "CVE-2021-44228" + + verify = subprocess.run([ + sys.executable, "-c", + "import json,sys,perseus; raw=open(sys.argv[3],'rb').read(); lineage=json.load(open(sys.argv[1])); query=json.load(open(sys.argv[2])); print(json.dumps([perseus.verify_sbom_lineage(lineage, raw_documents=[raw]), perseus.verify_sbom_lineage_query(query, lineage, [raw])]))", + str(lineage_path), str(query_path), str(raw_path), + ], capture_output=True, text=True, check=False, cwd=str(ROOT)) + assert verify.returncode == 0, verify.stdout + verify.stderr + checks = json.loads(verify.stdout) + assert checks[0]["valid"] is True + assert checks[1]["valid"] is True + + +def test_cli_query_rejects_oversized_raw_document_list_before_opening_lineage(tmp_path, monkeypatch, capsys): + opened = [] + + def unexpected_read(*args, **kwargs): + opened.append(args[0] if args else None) + raise AssertionError("lineage was opened before raw-document bound validation") + + monkeypatch.setattr(Path, "read_bytes", unexpected_read) + args = type("Args", (), { + "sbom_command": "query", "lineage": str(tmp_path / "missing-lineage.json"), "component": "x", + "raw_documents": [str(tmp_path / f"raw-{index}.json") for index in range(perseus._SL_MAX_DOCUMENTS + 1)], + "limit": 32, "output": None, "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 1 + assert opened == [] + failure = json.loads(capsys.readouterr().out) + assert failure["valid"] is False + assert "raw_documents" in failure["error"] + + +def test_privacy_sanitizer_covers_fragments_userinfo_markerless_private_refs_and_encoded_variants(): + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["type"] = "Bearer:RAW_BEARER_COLON" + component["bom-ref"] = "artifact:apikey/RAW_ID_SECRET" + component["purl"] = "pkg:generic/private@1?query=RAW_PURL_SECRET#RAW_FRAGMENT_SECRET" + component["externalReferences"] = [ + {"type": "website", "url": "https://user@example.invalid/public"}, + {"type": "website", "url": "https://private.example/home/public"}, + {"type": "website", "url": "https://example.invalid/basic:RAW_BASIC_SECRET"}, + {"type": "website", "url": "https://example.invalid/%61pikey/RAW_ENCODED_SECRET"}, + ] + document = perseus.ingest_sbom_document( + payload, + source_ref="artifact:%23RAW_SOURCE_FRAGMENT#RAW_SOURCE_HASH", + ) + serialized = json.dumps(document, sort_keys=True) + for marker in ( + "RAW_BEARER_COLON", "RAW_ID_SECRET", "RAW_PURL_SECRET", "RAW_FRAGMENT_SECRET", + "RAW_BASIC_SECRET", "RAW_ENCODED_SECRET", "RAW_SOURCE_FRAGMENT", "RAW_SOURCE_HASH", + ): + assert marker not in serialized + + xml = _load("spdx-app.xml").decode("utf-8").replace( + "Organization: Perseus Computing LLC", "Bearer:RAW_XML_SUPPLIER", + ) + xml_document = perseus.ingest_sbom_document(xml) + assert "RAW_XML_SUPPLIER" not in json.dumps(xml_document, sort_keys=True) + + +def test_spdx_document_id_namespace_and_component_relationship_rebinding_are_consistent(): + payload = json.loads(_load("spdx-app.json")) + payload["SPDXID"] = "SPDXRef-DOCUMENT-custom" + payload["packages"][0]["SPDXID"] = "artifact:reserved-component" + payload["relationships"] = [ + {"spdxElementId": "SPDXRef-DOCUMENT-custom", "relationshipType": "DESCRIBES", "relatedSpdxElement": "artifact:reserved-component"}, + ] + document = perseus.ingest_sbom_document(payload) + component_id = next(item["component_id"] for item in document["components"] if item["name"] == "log4j-core") + assert component_id.startswith("component:sha256:") + assert document["relationships"][0]["from"] == "SPDXRef-DOCUMENT-custom" + assert document["relationships"][0]["to"] == component_id + assert perseus.verify_sbom_document(document)["valid"] is True + + collision = json.loads(_load("spdx-app.json")) + collision["packages"][0]["SPDXID"] = "SPDXRef-DOCUMENT" + collision_document = perseus.ingest_sbom_document(collision) + assert collision_document["document_id"] not in {item["component_id"] for item in collision_document["components"]} + assert perseus.verify_sbom_document(collision_document)["valid"] is True + + invalid_namespace = json.loads(_load("spdx-app.json")) + invalid_namespace["SPDXID"] = "SPDXRef-NOT-DOCUMENT" + with pytest.raises(perseus.SBOMLineageError, match="document namespace"): + perseus.ingest_sbom_document(invalid_namespace) + + +def test_cyclonedx_reserved_component_ids_rewrite_dependency_endpoints(): + payload = json.loads(_load("cyclonedx-app.json")) + reserved = payload["components"][0] + reserved["bom-ref"] = "artifact:reserved-cdx" + payload["dependencies"] = [{ + "ref": "artifact:reserved-cdx", + "dependsOn": [reserved["purl"]], + }] + document = perseus.ingest_sbom_document(payload) + component_ids = {item["component_id"] for item in document["components"]} + assert any(item.startswith("component:sha256:") for item in component_ids) + assert document["relationships"] + assert all(edge["from"] in component_ids and edge["to"] in component_ids for edge in document["relationships"]) + assert not document["coverage"]["dangling_relationships"] + + +@pytest.mark.parametrize("serial_number", [None, False, 0, [], {}]) +def test_present_cyclonedx_serial_number_must_be_a_string(serial_number): + payload = json.loads(_load("cyclonedx-app.json")) + payload["serialNumber"] = serial_number + with pytest.raises(perseus.SBOMLineageError, match="serialNumber.*string|serialNumber.*required|ID"): + perseus.ingest_sbom_document(payload) + + +def test_json_exponent_overflow_is_rejected_even_in_ignored_fields(): + raw = _load("cyclonedx-app.json").decode("utf-8").replace( + '"version": 1,', '"version": 1, "ignored": {"overflow": 1e999},', + ) + with pytest.raises(perseus.SBOMLineageError, match="finite|non-finite|number"): + perseus.ingest_sbom_document(raw) + + +def test_bounded_reader_uses_a_descriptor_not_path_read_bytes(tmp_path, monkeypatch): + path = tmp_path / "small.json" + path.write_bytes(b"{}") + + def unexpected_read(*args, **kwargs): + raise AssertionError("bounded reader used Path.read_bytes") + + monkeypatch.setattr(Path, "read_bytes", unexpected_read) + assert perseus._sl_read_bounded(path) == b"{}" + + +def test_bounded_reader_rejects_non_regular_inputs_before_opening(tmp_path, monkeypatch): + path = tmp_path / "special" + os.mkfifo(path) + monkeypatch.setattr(Path, "read_bytes", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("special file was opened"))) + with pytest.raises(perseus.SBOMLineageError, match="regular"): + perseus._sl_read_bounded(path) + + +def test_bounded_reader_caps_a_read_race(tmp_path, monkeypatch): + path = tmp_path / "race.json" + path.write_bytes(b"x") + + class SmallStat: + st_size = 1 + st_mode = 0o100600 + + monkeypatch.setattr(perseus._sl_os, "lstat", lambda _path: SmallStat()) + monkeypatch.setattr(perseus._sl_os, "fstat", lambda _fd: SmallStat()) + monkeypatch.setattr(perseus._sl_os, "read", lambda _fd, size: b"x" * (size + 1)) + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_read_bounded(path) + + +def test_duplicate_json_object_keys_are_rejected_before_projection(): + raw = b'{"bomFormat":"CycloneDX","bomFormat":"SPDX","specVersion":"1.5"}' + with pytest.raises(perseus.SBOMLineageError, match="duplicate"): + perseus.ingest_sbom_document(raw) From e5c1b8aa9f38e6529c39e95c442e58310c5b8ba1 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 20:14:10 +0000 Subject: [PATCH 06/40] build(sbom): refresh generated artifact --- perseus.py | 342 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 271 insertions(+), 71 deletions(-) diff --git a/perseus.py b/perseus.py index 296bdd35..5d5a1acf 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "17f53e2-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "9421bbc" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44446,7 +44446,10 @@ def cmd_code_map(args, cfg) -> int: import hashlib import json +import math +import os as _sl_os import re +import stat as _sl_stat import xml.etree.ElementTree as _sl_et from urllib.parse import unquote from collections import deque @@ -44466,7 +44469,9 @@ def cmd_code_map(args, cfg) -> int: _SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") -_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer\s+|basic\s+|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*=|credential\s*=)") +_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=)") +_SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw)(?:[/?:=&_.-]|$)") +_SL_PUBLIC_PURL_QUERY_KEYS = frozenset({"classifier", "extension", "type", "repository_url"}) _SL_REFERENCE_TYPES = frozenset({ "advisory", "attestation", "cve", "distribution", "documentation", "license", "purl", "signature", "vex", "vulnerability", "website", "other", @@ -44518,7 +44523,7 @@ def cmd_code_map(args, cfg) -> int: # is caller-recomputable; only a projection produced by raw ingestion (or one # explicitly re-verified against raw bytes) is accepted by lineage builders. _SL_INGESTED_PROVENANCE: dict[str, tuple[str, str]] = {} -_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str]] = {} +_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str, str]] = {} class SBOMLineageError(ValueError): @@ -44529,6 +44534,19 @@ def _sl_json(value: Any) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) +def _sl_json_object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise SBOMLineageError("duplicate JSON object key") + result[key] = value + return result + + +def _sl_reject_json_constant(value: str) -> None: + raise SBOMLineageError(f"non-finite JSON constant is not allowed: {value}") + + def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() @@ -44538,6 +44556,29 @@ def _sl_ingestion_sha(unsigned: Mapping[str, Any], raw_bytes: bytes | None = Non return _sl_sha({"domain": "perseus-sbom-ingestion", "raw_sha256": raw_digest, "projection": unsigned}) +def _sl_private_locator(value: str) -> bool: + """Identify private/credential-bearing URI and identifier shapes.""" + decoded = value + for _ in range(min(len(value) + 1, 1025)): + next_decoded = unquote(decoded) + if next_decoded == decoded: + break + decoded = next_decoded + lowered = decoded.casefold() + if "#" in decoded or lowered.startswith("file:") or _SL_PRIVATE_LOCATOR_RE.search(decoded): + return True + authority = decoded.split("://", 1)[1].split("/", 1)[0] if "://" in decoded else "" + if authority and "@" in authority: + return True + if "?" in decoded: + query = decoded.split("?", 1)[1].split("#", 1)[0] + for pair in query.split("&"): + key = pair.split("=", 1)[0].casefold() + if key and key not in _SL_PUBLIC_PURL_QUERY_KEYS: + return True + return False + + def _sl_sensitive(value: str) -> bool: """Return whether a scalar looks like it carries a credential.""" candidates = [value] @@ -44552,11 +44593,10 @@ def _sl_sensitive(value: str) -> bool: candidates.append(next_decoded) decoded = next_decoded for candidate in candidates: - authority = candidate.split("://", 1)[1].split("/", 1)[0] if "://" in candidate else "" normalized = re.sub(r"[^a-z0-9]+", "_", candidate.casefold()) if ( _SL_SENSITIVE_REFERENCE_RE.search(candidate) - or (authority and "@" in authority) + or _sl_private_locator(candidate) or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) ): return True @@ -44597,7 +44637,7 @@ def _sl_text(value: Any, field: str, *, required: bool = False, limit: int = 512 raise SBOMLineageError(f"{field} is required") return "" if not isinstance(value, str): - value = str(value) + raise SBOMLineageError(f"{field} must be a string") text = re.sub(r"[\x00-\x1f\x7f]", " ", value).strip() if required and not text: raise SBOMLineageError(f"{field} is required") @@ -44633,8 +44673,10 @@ def _sl_safe_locator(value: Any, field: str) -> str: def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: - if value in (None, ""): + if value == "": return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" + if not isinstance(value, str): + raise SBOMLineageError("source_ref must be a string") text = _sl_text(value, "source_ref", required=True, limit=512) raw_digest = hashlib.sha256(raw_bytes).hexdigest() if text.startswith("sha256:source-ref:"): @@ -44887,10 +44929,17 @@ def _sl_component( component_type: Any = None, licenses: Any = None, truncated: list[str] | None = None, + component_id_map: dict[str, str] | None = None, ) -> dict[str, Any]: normalized_name = _sl_safe_text(name, "component_name", required=True, limit=256) normalized_version = _sl_safe_text(version, "component_version", limit=128) normalized_id = _sl_id(component_id, "component ID") + if isinstance(component_id, str) and component_id_map is not None: + component_id_map[component_id] = normalized_id + if _sl_node_kind(normalized_id) != "component": + normalized_id = "component:sha256:" + hashlib.sha256(normalized_id.encode("utf-8")).hexdigest() + if isinstance(component_id, str) and component_id_map is not None: + component_id_map[component_id] = normalized_id ids = {normalized_id} component_truncated: list[str] = list(truncated or []) if identifiers is not None and not isinstance(identifiers, (list, tuple, set)): @@ -44902,6 +44951,9 @@ def _sl_component( text = _sl_safe_locator(identifier, "component_identifier") if text: ids.add(text) + if component_id_map is not None: + for identifier in ids: + component_id_map.setdefault(identifier, normalized_id) safe_references = [] if references is not None and not isinstance(references, (list, tuple)): raise SBOMLineageError("component references must be a list") @@ -44950,12 +45002,17 @@ def _sl_component( "identifiers": sorted(ids), "references": sorted(safe_references, key=lambda item: (item.get("type", ""), item.get("locator", ""))), "licenses": sorted(set(license_values)), - "component_type": _sl_text(component_type, "component_type", limit=64) or "unknown", + "component_type": _sl_safe_text(component_type, "component_type", limit=128) or "unknown", "coverage": {"state": coverage_state, "unknown": sorted(set(unknown)), "truncated": sorted(set(component_truncated))}, } -def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None, component_id_map: Mapping[str, str] | None = None) -> dict[str, Any]: + if component_id_map is not None: + if isinstance(source, str): + source = component_id_map.get(source, source) + if isinstance(target, str): + target = component_id_map.get(target, target) source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") @@ -44982,7 +45039,7 @@ def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confid return result -def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] references, identifiers = _sl_spdx_references(raw.get("externalRefs"), truncated=component_truncated) if truncated is not None: @@ -44997,10 +45054,11 @@ def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = component_type="package", licenses=raw.get("licenseConcluded"), truncated=component_truncated, + component_id_map=component_id_map, ) -def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] references, identifiers = _sl_cdx_references(raw.get("externalReferences"), truncated=component_truncated) if "purl" in raw and raw.get("purl") is not None: @@ -45008,7 +45066,7 @@ def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = N references.extend(_sl_properties(raw.get("properties"), truncated=component_truncated)) if truncated is not None: truncated.extend(item for item in component_truncated if item not in truncated) - return _sl_component( + component = _sl_component( component_id=raw.get("bom-ref"), name=raw.get("name"), version=raw.get("version"), @@ -45018,7 +45076,11 @@ def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = N component_type=raw.get("type"), licenses=raw.get("licenses"), truncated=component_truncated, + component_id_map=component_id_map, ) + if component_id_map is not None and isinstance(raw.get("purl"), str): + component_id_map.setdefault(raw["purl"], component["component_id"]) + return component def _sl_validate_spdx_version(value: Any) -> str: @@ -45036,6 +45098,13 @@ def _sl_validate_cdx_version(value: Any) -> str: return version_text +def _sl_spdx_document_id(value: Any) -> str: + document_id = _sl_id(value, "SPDXID") + if _sl_node_kind(document_id) != "document": + raise SBOMLineageError("SPDXID must use the document namespace") + return document_id + + def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") @@ -45043,12 +45112,15 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu raise SBOMLineageError("normalized SBOM components exceed their global bound") if len(relationships) > _SL_MAX_RELATIONSHIPS: raise SBOMLineageError("normalized SBOM relationships exceed their global bound") + safe_supplier = _sl_safe_text(supplier, "supplier", limit=512) component_ids: set[str] = set() for item in components: component_id = item.get("component_id") if isinstance(item, Mapping) else None if not isinstance(component_id, str) or component_id in component_ids: raise SBOMLineageError("duplicate or missing component ID") component_ids.add(component_id) + if fmt == "SPDX" and document_id in component_ids: + raise SBOMLineageError("SPDX document ID collides with a component ID") known_ids = set(component_ids) if fmt == "SPDX": known_ids.add(document_id) @@ -45063,7 +45135,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("document_name") if not created_at: unknown.append("created_at") - if not supplier: + if not safe_supplier: unknown.append("supplier") if dangling: unknown.append("dangling_relationships") @@ -45073,7 +45145,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu unknown.append("component_metadata") if not components: coverage_state = "unknown" - elif dangling or truncation or not document_name or not created_at or not supplier or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: + elif dangling or truncation or not document_name or not created_at or not safe_supplier or any(item.get("coverage", {}).get("state") != "complete" for item in components) or not relationships: coverage_state = "partial" else: coverage_state = "complete" @@ -45086,7 +45158,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu "document_sha256": hashlib.sha256(raw_bytes).hexdigest(), "source_ref": source_ref, "created_at": _sl_safe_text(created_at, "created_at", limit=128) or None, - "supplier": supplier or None, + "supplier": safe_supplier or None, "components": sorted(components, key=lambda item: (item["component_id"] == metadata_component_id, item["component_id"])), "relationships": sorted(relationships, key=lambda item: (item["from"], item["to"], item["type"])), "coverage": { @@ -45108,7 +45180,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: str) -> dict[str, Any]: version = _sl_validate_spdx_version(value.get("spdxVersion")) - document_id = _sl_id(value.get("SPDXID"), "SPDXID") + document_id = _sl_spdx_document_id(value.get("SPDXID")) creation = value.get("creationInfo", {}) if creation is None: creation = {} @@ -45125,18 +45197,19 @@ def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: truncated.append("packages") if len(raw_relationships) > _SL_MAX_RELATIONSHIPS: truncated.append("relationships") + component_id_map: dict[str, str] = {} components = [] for item in packages[:_SL_MAX_COMPONENTS]: if not isinstance(item, Mapping): raise SBOMLineageError("packages must contain objects") - components.append(_sl_spdx_component(item, truncated=truncated)) + components.append(_sl_spdx_component(item, truncated=truncated, component_id_map=component_id_map)) relationships = [] for raw in raw_relationships[:_SL_MAX_RELATIONSHIPS]: if not isinstance(raw, Mapping): raise SBOMLineageError("relationships must contain objects") relationships.append(_sl_relationship( raw.get("spdxElementId"), raw.get("relatedSpdxElement"), - raw.get("relationshipType", "related"), truncated=truncated, + raw.get("relationshipType", "related"), truncated=truncated, component_id_map=component_id_map, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -45174,8 +45247,9 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s raise SBOMLineageError("components must contain objects") raw_components.append(item) components: list[dict[str, Any]] = [] + component_id_map: dict[str, str] = {} for raw in raw_components: - components.append(_sl_cdx_component(raw, truncated=truncated)) + components.append(_sl_cdx_component(raw, truncated=truncated, component_id_map=component_id_map)) relationships = [] for raw in raw_dependency_list[:_SL_MAX_RELATIONSHIPS]: if not isinstance(raw, Mapping): @@ -45192,12 +45266,15 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s if len(targets) > allowed_targets: truncated.append("dependency_edges") for target in targets[:allowed_targets]: - relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated)) + relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated, component_id_map=component_id_map)) creators = metadata.get("authors", []) if not isinstance(creators, list): raise SBOMLineageError("metadata.authors must be a list") supplier = _sl_supplier(creators[0] if creators else (metadata_component or {})) - document_id = _sl_id(value.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + if "serialNumber" in value: + document_id = _sl_id(value.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_text(metadata.get("name"), "document_name"), created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, @@ -45213,7 +45290,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") if not raw_document_id: raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") - document_id = _sl_id(raw_document_id, "SPDXID") + document_id = _sl_spdx_document_id(raw_document_id) creation = next(iter(_sl_descendants(document, "creationInfo")), None) created_at = _sl_xml_value(creation, "created") creator = _sl_xml_value(creation, "creator") @@ -45224,6 +45301,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, truncated.append("packages") if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: truncated.append("relationships") + component_id_map: dict[str, str] = {} components = [] for raw in packages[:_SL_MAX_COMPONENTS]: component_truncated: list[str] = [] @@ -45254,7 +45332,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, references=refs, component_type="package", licenses=_sl_xml_value(raw, "licenseConcluded"), - truncated=component_truncated, + truncated=component_truncated, component_id_map=component_id_map, )) relationships = [] for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: @@ -45262,7 +45340,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, _sl_xml_value(raw, "spdxElementId"), _sl_xml_value(raw, "relatedSpdxElement"), _sl_xml_value(raw, "relationshipType", default="related"), - truncated=truncated, + truncated=truncated, component_id_map=component_id_map, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -45273,7 +45351,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) - document_id = _sl_id(_sl_xml_text(root, "SPDXID"), "SPDXID") + document_id = _sl_spdx_document_id(_sl_xml_text(root, "SPDXID")) creation = _sl_child(root, "creationInfo") created_at = _sl_xml_text(creation, "created") if creation is not None else "" creator = _sl_xml_text(creation, "creator") if creation is not None else "" @@ -45284,6 +45362,7 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] truncated.append("packages") if len(relationships_raw) > _SL_MAX_RELATIONSHIPS: truncated.append("relationships") + component_id_map: dict[str, str] = {} components = [] for raw in packages[:_SL_MAX_COMPONENTS]: component_truncated: list[str] = [] @@ -45307,14 +45386,14 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] version=_sl_xml_text(raw, "versionInfo"), supplier=_sl_xml_text(raw, "supplier"), identifiers=identifiers, references=refs, component_type="package", licenses=_sl_xml_text(raw, "licenseConcluded"), - truncated=component_truncated, + truncated=component_truncated, component_id_map=component_id_map, )) relationships = [] for raw in relationships_raw[:_SL_MAX_RELATIONSHIPS]: relationships.append(_sl_relationship( _sl_xml_text(raw, "spdxElementId"), _sl_xml_text(raw, "relatedSpdxElement"), _sl_xml_text(raw, "relationshipType", default="related"), - truncated=truncated, + truncated=truncated, component_id_map=component_id_map, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -45323,7 +45402,7 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] ) -def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None) -> dict[str, Any]: +def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] refs = [] identifiers = [] @@ -45353,12 +45432,15 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None) -> di licenses.append(_sl_xml_text(item, "id") or _sl_xml_text(item, "name")) if truncated is not None: truncated.extend(item for item in component_truncated if item not in truncated) - return _sl_component( + component = _sl_component( component_id=raw.attrib.get("bom-ref"), name=_sl_xml_text(raw, "name"), version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_child(raw, "supplier"), "name"), identifiers=identifiers, references=refs, component_type=raw.attrib.get("type"), licenses=licenses, - truncated=component_truncated, + truncated=component_truncated, component_id_map=component_id_map, ) + if component_id_map is not None and purl: + component_id_map.setdefault(purl, component["component_id"]) + return component def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: @@ -45380,9 +45462,10 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if len(component_nodes) > component_budget: truncated.append("components") raw_components.extend(component_nodes[:component_budget]) + component_id_map: dict[str, str] = {} components = [] for raw in raw_components: - components.append(_sl_cdx_xml_component(raw, truncated=truncated)) + components.append(_sl_cdx_xml_component(raw, truncated=truncated, component_id_map=component_id_map)) relationships = [] dependencies_node = _sl_child(root, "dependencies") if dependencies_node is not None: @@ -45402,11 +45485,14 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if len(children) > allowed_targets: truncated.append("dependency_edges") for child in children[:allowed_targets]: - relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated)) + relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated, component_id_map=component_id_map)) timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" authors = _sl_child(metadata, "authors") if metadata is not None else None author = _sl_xml_text(_sl_child(authors, "author"), "name") if authors is not None else "" - document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber", fallback="document:cyclonedx") + if "serialNumber" in root.attrib: + document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, @@ -45416,20 +45502,42 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_read_bounded(path: Path) -> bytes: - """Read a file only after checking its size, including race re-check.""" - try: - size = path.stat().st_size + """Read a regular file through a no-follow descriptor with a hard cap.""" + try: + initial = _sl_os.lstat(path) + if not _sl_stat.S_ISREG(initial.st_mode): + raise SBOMLineageError("SBOM input must be a regular file") + flags = _sl_os.O_RDONLY | getattr(_sl_os, "O_CLOEXEC", 0) | getattr(_sl_os, "O_NOFOLLOW", 0) | getattr(_sl_os, "O_NONBLOCK", 0) + fd = _sl_os.open(path, flags) + except SBOMLineageError: + raise except OSError as exc: - raise SBOMLineageError(f"could not stat SBOM: {exc}") from exc - if size > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") - try: - raw_bytes = path.read_bytes() + raise SBOMLineageError(f"could not open SBOM safely: {exc}") from exc + try: + opened = _sl_os.fstat(fd) + if not _sl_stat.S_ISREG(opened.st_mode): + raise SBOMLineageError("SBOM input must be a regular file") + initial_identity = (getattr(initial, "st_dev", None), getattr(initial, "st_ino", None)) + opened_identity = (getattr(opened, "st_dev", None), getattr(opened, "st_ino", None)) + if None not in initial_identity + opened_identity and opened_identity != initial_identity: + raise SBOMLineageError("SBOM input changed during safe open") + if opened.st_size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + data = bytearray() + while len(data) <= _SL_MAX_INPUT_BYTES: + chunk = _sl_os.read(fd, min(64 * 1024, _SL_MAX_INPUT_BYTES + 1 - len(data))) + if not chunk: + break + data.extend(chunk) + if len(data) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return bytes(data) + except SBOMLineageError: + raise except OSError as exc: - raise SBOMLineageError(f"could not read SBOM: {exc}") from exc - if len(raw_bytes) > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") - return raw_bytes + raise SBOMLineageError(f"could not read SBOM safely: {exc}") from exc + finally: + _sl_os.close(fd) def _sl_validate_json_depth(raw_bytes: bytes) -> None: @@ -45455,12 +45563,30 @@ def _sl_validate_json_depth(raw_bytes: bytes) -> None: depth = max(0, depth - 1) +def _sl_validate_json_values(value: Any) -> None: + stack = [value] + while stack: + current = stack.pop() + if isinstance(current, float) and not math.isfinite(current): + raise SBOMLineageError("non-finite JSON number is not allowed") + if isinstance(current, Mapping): + stack.extend(current.values()) + elif isinstance(current, list): + stack.extend(current) + + def _sl_load_json_bounded(source: Path | bytes, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: """Load JSON through the shared byte and nesting bounds.""" raw_bytes = _sl_read_bounded(source) if isinstance(source, Path) else bytes(source) _sl_validate_json_depth(raw_bytes) try: - return json.loads(raw_bytes.decode("utf-8")), raw_bytes + value = json.loads( + raw_bytes.decode("utf-8"), + object_pairs_hook=_sl_json_object_pairs, + parse_constant=_sl_reject_json_constant, + ) + _sl_validate_json_values(value) + return value, raw_bytes except RecursionError as exc: raise SBOMLineageError("SBOM JSON nesting is too deep") from exc except (UnicodeDecodeError, json.JSONDecodeError) as exc: @@ -45526,7 +45652,12 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raise SBOMLineageError("SBOM input is empty") try: _sl_validate_json_depth(raw_bytes) - value = json.loads(raw_bytes.decode("utf-8")) + value = json.loads( + raw_bytes.decode("utf-8"), + object_pairs_hook=_sl_json_object_pairs, + parse_constant=_sl_reject_json_constant, + ) + _sl_validate_json_values(value) if not isinstance(value, Mapping): raise SBOMLineageError("SBOM JSON root must be an object") return dict(value), raw_bytes @@ -45563,7 +45694,7 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An def _sl_node_kind(node_id: str) -> str: - if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("document:"): + if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("SPDXRef-DOCUMENT-") or node_id.startswith("document:"): return "document" if node_id.startswith("artifact:"): return "artifact" @@ -45663,7 +45794,7 @@ def _sl_validate_component(component: Any) -> dict[str, Any]: raise SBOMLineageError("component.references exceeds its bound") checked_references = [_sl_validate_reference(reference) for reference in references] licenses = _sl_string_list(component.get("licenses"), "component.licenses", maximum=_SL_MAX_LICENSES) - component_type = _sl_strict_text(component.get("component_type"), "component.component_type", limit=64) + component_type = _sl_strict_text(component.get("component_type"), "component.component_type", limit=128) assert component_type is not None coverage = _sl_validate_component_coverage( component.get("coverage"), has_version=version is not None, has_supplier=supplier is not None, @@ -45780,6 +45911,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non document_id_text = _sl_strict_text(document.get("document_id"), "document_id", limit=256) assert document_id_text is not None document_id = _sl_id(document_id_text, "document_id") + if fmt == "SPDX" and _sl_node_kind(document_id) != "document": + raise SBOMLineageError("SPDX document ID must use the document namespace") for field, limit in (("document_name", 256), ("created_at", 128), ("supplier", 512)): _sl_strict_text(document.get(field), field, allow_none=True, limit=limit) document_sha = document.get("document_sha256") @@ -45800,6 +45933,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non raise SBOMLineageError("normalized SBOM contains duplicate component IDs") seen.add(checked["component_id"]) checked_components.append(checked) + if fmt == "SPDX" and document_id in seen: + raise SBOMLineageError("SPDX document ID collides with a component ID") checked_relationships = [_sl_validate_relationship(item) for item in relationships] coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") @@ -45957,6 +46092,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any } body["lineage_digest"] = _sl_sha(body) _SL_LINEAGE_PROVENANCE[body["lineage_digest"]] = ( + _sl_json(body), _sl_json(body["nodes"]), _sl_json(body["edges"]), ) @@ -46011,7 +46147,7 @@ def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[ return node_id -def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: +def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None = None) -> dict[str, Any]: required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: raise SBOMLineageError("unsupported software-lineage schema") @@ -46020,12 +46156,22 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: unsigned.pop("lineage_digest", None) if not isinstance(supplied, str) or not re.fullmatch(r"[0-9a-f]{64}", supplied) or _sl_sha(unsigned) != supplied: raise SBOMLineageError("lineage digest mismatch") + trusted = _SL_LINEAGE_PROVENANCE.get(supplied) + if raw_documents is None and (trusted is None or trusted[0] != _sl_json(dict(lineage))): + raise SBOMLineageError("lineage is not bound to a trusted builder receipt or raw documents") documents = _sl_list(lineage.get("documents"), "lineage.documents") nodes = _sl_list(lineage.get("nodes"), "lineage.nodes") edges = _sl_list(lineage.get("edges"), "lineage.edges") + if not documents: + raise SBOMLineageError("lineage must contain at least one document") if len(documents) > _SL_MAX_DOCUMENTS or len(edges) > _SL_MAX_EDGES or len(nodes) > _SL_MAX_NODES: raise SBOMLineageError("lineage collection exceeds its bound") - checked_documents = [_sl_validate_document(document) for document in documents] + if raw_documents is None: + checked_documents = [_sl_validate_document(document) for document in documents] + else: + if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: + raise SBOMLineageError("raw_documents must match lineage documents within its bound") + checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} for document in checked_documents: for component in document["components"]: @@ -46075,9 +46221,9 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: return dict(lineage) -def verify_sbom_lineage(lineage: Mapping[str, Any]) -> dict[str, Any]: +def verify_sbom_lineage(lineage: Mapping[str, Any], raw_documents: Any | None = None) -> dict[str, Any]: try: - loaded = _sl_loaded_lineage(lineage) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -46118,9 +46264,9 @@ def _sl_path_confidence(path: list[Mapping[str, Any]]) -> str: return "high" -def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32) -> dict[str, Any]: +def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32, raw_documents: Any | None = None) -> dict[str, Any]: """Find impacted artifact nodes and return every traversed evidence edge.""" - loaded = _sl_loaded_lineage(lineage) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) limit = _sl_limit(limit, "limit") query_text = _sl_strict_text(query, "query", limit=512) assert query_text is not None @@ -46213,6 +46359,7 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 "lineage_nodes": loaded["nodes"], "lineage_edges": loaded["edges"], "query": query_text, + "limit": limit, "matched_nodes": matches, "impacted_artifacts": impacted, "status": status, @@ -46223,8 +46370,8 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 return unsigned -def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: - required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} +def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: + required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") supplied = query_result.get("query_digest") @@ -46241,12 +46388,21 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query authoritative lineage exceeds its bound") authority = _SL_LINEAGE_PROVENANCE.get(lineage_digest) if authoritative_lineage is not None: - loaded_authority = _sl_loaded_lineage(authoritative_lineage) + loaded_authority = _sl_loaded_lineage(authoritative_lineage, raw_documents=raw_documents) if lineage_digest != loaded_authority["lineage_digest"]: raise SBOMLineageError("query result lineage digest does not match the authoritative lineage") - authority = (_sl_json(loaded_authority["nodes"]), _sl_json(loaded_authority["edges"])) - if authority is None or authority != (_sl_json(lineage_nodes), _sl_json(lineage_edges)): + authority = ( + _sl_json(loaded_authority), + _sl_json(loaded_authority["nodes"]), + _sl_json(loaded_authority["edges"]), + ) + if authority is None or authority[1:] != (_sl_json(lineage_nodes), _sl_json(lineage_edges)): raise SBOMLineageError("query result is not bound to the authoritative lineage digest and nodes/edges") + if authoritative_lineage is None: + try: + authoritative_lineage = json.loads(authority[0]) + except (TypeError, json.JSONDecodeError) as exc: + raise SBOMLineageError("authoritative lineage receipt is malformed") from exc authority_node_ids: set[str] = set() for node in lineage_nodes: if not isinstance(node, Mapping): @@ -46273,6 +46429,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin authority_edges.append(checked_edge) query = _sl_strict_text(query_result.get("query"), "query", limit=512) assert query is not None + limit = _sl_limit(query_result.get("limit"), "query.limit") matched_nodes = _sl_string_list(query_result.get("matched_nodes"), "matched_nodes", maximum=_SL_MAX_QUERY_MATCHES) for node_id in matched_nodes: _sl_id(node_id, "matched_node") @@ -46357,17 +46514,46 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query claims are inconsistent with coverage") if status != expected_status: raise SBOMLineageError("query status is inconsistent with coverage") + assert authoritative_lineage is not None + expected_result = query_sbom_lineage( + authoritative_lineage, query, limit=limit, raw_documents=raw_documents, + ) + for field in ( + "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", + "impacted_artifacts", "status", "coverage", "claims", "query_digest", + ): + if query_result.get(field) != expected_result.get(field): + raise SBOMLineageError(f"query field {field} is inconsistent with authoritative recomputation") return dict(query_result) -def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None) -> dict[str, Any]: +def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: try: - checked = _sl_validate_query_result(query_result, authoritative_lineage) + checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": str(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} +def _sl_cli_bounded_paths(value: Any, field: str, *, required: bool = False) -> list[str] | None: + if value is None: + if required: + raise SBOMLineageError(f"{field} is required") + return None + if not isinstance(value, (list, tuple)): + raise SBOMLineageError(f"{field} must be a list") + if not value: + raise SBOMLineageError(f"{field} must not be empty") + if len(value) > _SL_MAX_DOCUMENTS: + raise SBOMLineageError(f"{field} exceeds {_SL_MAX_DOCUMENTS} documents") + result: list[str] = [] + for item in value: + if not isinstance(item, str) or not item: + raise SBOMLineageError(f"{field} paths must be strings") + result.append(item) + return result + + def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: """CLI adapter for local SBOM normalization, merge, and query operations.""" try: @@ -46376,17 +46562,18 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: if command == "ingest": document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) elif command == "merge": + document_paths = _sl_cli_bounded_paths(getattr(args, "documents", None), "documents", required=True) + raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") + if raw_document_paths is not None and len(raw_document_paths) != len(document_paths or []): + raise SBOMLineageError("raw_documents must contain one raw source per document") documents = [] - for path in args.documents: + for path in document_paths or []: payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) - raw_documents = getattr(args, "raw_documents", None) raw_inputs = None - if raw_documents is not None: - if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents): - raise SBOMLineageError("raw_documents must contain one raw source per document") + if raw_document_paths is not None: raw_inputs = [] - for raw_path in raw_documents: + for raw_path in raw_document_paths: _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) raw_inputs.append(raw_bytes) if getattr(args, "edges", None): @@ -46397,10 +46584,19 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: edges = [] document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) elif command == "query": + raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage, _ = _sl_load_json_bounded(Path(args.lineage)) if not isinstance(lineage, Mapping): raise SBOMLineageError("lineage JSON root must be an object") - document = query_sbom_lineage(lineage, args.component, limit=getattr(args, "limit", 32)) + raw_inputs = None + if raw_document_paths is not None: + lineage_documents = lineage.get("documents") + if not isinstance(lineage_documents, list) or len(raw_document_paths) != len(lineage_documents): + raise SBOMLineageError("raw_documents must contain one raw source per lineage document") + raw_inputs = [_sl_load_json_bounded(Path(path), allow_non_json=True)[1] for path in raw_document_paths] + document = query_sbom_lineage( + lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, + ) else: raise SBOMLineageError("command must be ingest, merge, or query") serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" @@ -46413,7 +46609,10 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: print(f"sbom {command} -> {output}\n{digest_key}: {document[digest_key]}") return 0 except (OSError, TypeError, ValueError, SBOMLineageError) as exc: - print(f"sbom: {exc}") + if getattr(args, "json", False): + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": str(exc), "valid": False}, sort_keys=True)) + else: + print(f"sbom: {exc}") return 1 # ───────────────────── Prompt-size forensics (#606) ────────────────────────── # @@ -47270,7 +47469,7 @@ def main(): p_sbom_ingest.add_argument("--json", action="store_true", help="Print machine-readable JSON") p_sbom_merge = sbom_sub.add_parser("merge", help="Build a queryable lineage graph from normalized SBOM documents") p_sbom_merge.add_argument("documents", nargs="+", help="SBOM document paths") - p_sbom_merge.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to normalized documents") + p_sbom_merge.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to normalized documents; required for persisted normalized inputs") p_sbom_merge.add_argument("--edges", default=None, help="Optional JSON file of source/build/artifact/deployment edges") p_sbom_merge.add_argument("--output", "-o", default=None, help="Write the lineage graph to a JSON file") p_sbom_merge.add_argument("--json", action="store_true", help="Print machine-readable JSON") @@ -47278,6 +47477,7 @@ def main(): p_sbom_query.add_argument("lineage", help="Lineage graph JSON path") p_sbom_query.add_argument("component", help="Component name, version, purl, or vulnerability reference") p_sbom_query.add_argument("--limit", type=int, default=32, help="Maximum impacted artifacts") + p_sbom_query.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to lineage documents; required across processes") p_sbom_query.add_argument("--output", "-o", default=None, help="Write the query result to a JSON file") p_sbom_query.add_argument("--json", action="store_true", help="Print machine-readable JSON") From 419420b93962d29d1842307ae89f19501fa73c52 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 20:18:26 +0000 Subject: [PATCH 07/40] fix(sbom): reject explicit null PURLs --- src/perseus/sbom_lineage.py | 2 +- tests/test_sbom_lineage.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 7da0ab6f..4d41508c 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -625,7 +625,7 @@ def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] references, identifiers = _sl_cdx_references(raw.get("externalReferences"), truncated=component_truncated) - if "purl" in raw and raw.get("purl") is not None: + if "purl" in raw: identifiers.append(_sl_safe_locator(raw.get("purl"), "purl")) references.extend(_sl_properties(raw.get("properties"), truncated=component_truncated)) if truncated is not None: diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 13641f91..47b81b61 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -709,6 +709,10 @@ def test_component_and_reference_identifier_fields_reject_non_strings(): payload["components"][0]["purl"] = 123 cases.append(payload) + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["purl"] = None + cases.append(payload) + payload = json.loads(_load("cyclonedx-app.json")) payload["components"][0]["externalReferences"] = [{"type": "website", "url": 123}] cases.append(payload) From 5beb48f635e77ddf91fe21415dda9d9408426e47 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 20:18:27 +0000 Subject: [PATCH 08/40] build(sbom): refresh generated artifact --- perseus.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/perseus.py b/perseus.py index 5d5a1acf..bb97dcfd 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "9421bbc" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "11b06fa" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -45061,7 +45061,7 @@ def _sl_spdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = None, component_id_map: dict[str, str] | None = None) -> dict[str, Any]: component_truncated: list[str] = [] references, identifiers = _sl_cdx_references(raw.get("externalReferences"), truncated=component_truncated) - if "purl" in raw and raw.get("purl") is not None: + if "purl" in raw: identifiers.append(_sl_safe_locator(raw.get("purl"), "purl")) references.extend(_sl_properties(raw.get("properties"), truncated=component_truncated)) if truncated is not None: From 8d839d056af4cfb596ace7503ac7953cd19ea556 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 21:03:34 +0000 Subject: [PATCH 09/40] fix(sbom): close privacy parser and lineage follow-up --- src/perseus/sbom_lineage.py | 434 ++++++++++++++++++++++++++---------- tests/test_sbom_lineage.py | 199 ++++++++++++++++- 2 files changed, 516 insertions(+), 117 deletions(-) diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 4d41508c..48a667b5 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -33,8 +33,9 @@ _SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") -_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=)") +_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=|authorization\s*[/=:])") _SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw)(?:[/?:=&_.-]|$)") +_SL_PRIVACY_MARKER_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(api[_-]?key|authorization|password|passwd|secret|token|credential)(?:[/?:=&_.-]|$)") _SL_PUBLIC_PURL_QUERY_KEYS = frozenset({"classifier", "extension", "type", "repository_url"}) _SL_REFERENCE_TYPES = frozenset({ "advisory", "attestation", "cve", "distribution", "documentation", "license", @@ -94,6 +95,13 @@ class SBOMLineageError(ValueError): """Raised when an SBOM or lineage projection cannot be verified.""" +def _sl_public_error(exc: BaseException) -> str: + """Serialize only bounded, non-exception-derived public error text.""" + if isinstance(exc, SBOMLineageError): + return str(exc)[:160] + return "SBOM input is invalid" + + def _sl_json(value: Any) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) @@ -108,7 +116,7 @@ def _sl_json_object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: def _sl_reject_json_constant(value: str) -> None: - raise SBOMLineageError(f"non-finite JSON constant is not allowed: {value}") + raise SBOMLineageError("non-finite JSON constant is not allowed") def _sl_sha(value: Any) -> str: @@ -120,19 +128,47 @@ def _sl_ingestion_sha(unsigned: Mapping[str, Any], raw_bytes: bytes | None = Non return _sl_sha({"domain": "perseus-sbom-ingestion", "raw_sha256": raw_digest, "projection": unsigned}) -def _sl_private_locator(value: str) -> bool: - """Identify private/credential-bearing URI and identifier shapes.""" +def _sl_decoded_variants(value: str) -> list[str]: + """Return bounded, recursively percent-decoded inspection candidates.""" + candidates = [value] decoded = value for _ in range(min(len(value) + 1, 1025)): next_decoded = unquote(decoded) if next_decoded == decoded: break + candidates.append(next_decoded) decoded = next_decoded + return candidates + + +def _sl_userinfo_locator(value: str) -> bool: + """Detect URI and git-style userinfo without treating version ``@`` as auth.""" + for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): + if "@" in match.group(1): + return True + # ``git:user@host:repo`` and ``build:user@host/repo`` do not have ``://``. + # A versioned PURL/artifact ID ends in ``@version`` and has no locator-like + # suffix after the at-sign, so it remains a public identifier. + for at in (index for index, char in enumerate(value) if char == "@"): + suffix = value[at + 1:].split("#", 1)[0].split("?", 1)[0] + if "->" in suffix: + continue + if re.match(r"^[^/]+:[^/]+", suffix) or "/" in suffix: + return True + return False + + +def _sl_private_locator(value: str) -> bool: + """Identify private, credential-bearing, or non-public locator shapes.""" + decoded = _sl_decoded_variants(value)[-1] lowered = decoded.casefold() - if "#" in decoded or lowered.startswith("file:") or _SL_PRIVATE_LOCATOR_RE.search(decoded): - return True - authority = decoded.split("://", 1)[1].split("/", 1)[0] if "://" in decoded else "" - if authority and "@" in authority: + if ( + "#" in decoded + or lowered.startswith("file:") + or _SL_PRIVATE_LOCATOR_RE.search(decoded) + or _SL_PRIVACY_MARKER_RE.search(decoded) + or _sl_userinfo_locator(decoded) + ): return True if "?" in decoded: query = decoded.split("?", 1)[1].split("#", 1)[0] @@ -144,22 +180,12 @@ def _sl_private_locator(value: str) -> bool: def _sl_sensitive(value: str) -> bool: - """Return whether a scalar looks like it carries a credential.""" - candidates = [value] - decoded = value - # Decode until stable, bounded by the scalar length. Each successful - # percent-decoding pass must consume at least one encoded triplet, so this - # remains bounded for the 1024-character input limit. - for _ in range(min(len(value) + 1, 1025)): - next_decoded = unquote(decoded) - if next_decoded == decoded: - break - candidates.append(next_decoded) - decoded = next_decoded - for candidate in candidates: + """Return whether a scalar looks like it carries private material.""" + for candidate in _sl_decoded_variants(value): normalized = re.sub(r"[^a-z0-9]+", "_", candidate.casefold()) if ( _SL_SENSITIVE_REFERENCE_RE.search(candidate) + or _SL_PRIVACY_MARKER_RE.search(candidate) or _sl_private_locator(candidate) or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) ): @@ -335,11 +361,33 @@ def _sl_children(element: Any, name: str) -> list[Any]: return [child for child in list(element) if _sl_local(child) == name] +def _sl_single_child(element: Any, name: str) -> Any | None: + matches = _sl_children(element, name) + if len(matches) > 1: + raise SBOMLineageError("XML singleton element is duplicated") + return matches[0] if matches else None + + +def _sl_xml_scalar(element: Any, names: tuple[str, ...], *, default: str = "") -> str: + wanted = {name.casefold() for name in names} + values: list[str] = [] + for child in list(element) if element is not None else []: + if _sl_local(child).casefold() not in wanted: + continue + text = (child.text or "").strip() if child.text else "" + scalar = text + for key, value in child.attrib.items(): + if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: + scalar = str(value).lstrip("#") + break + values.append(scalar) + if values and any(value != values[0] for value in values[1:]): + raise SBOMLineageError("XML singleton values conflict") + return values[0] if values else default + + def _sl_xml_text(element: Any, name: str, *, default: str = "") -> str: - if element is None: - return default - child = _sl_child(element, name) - return (child.text or "").strip() if child is not None and child.text else default + return _sl_xml_scalar(element, (name,), default=default) def _sl_validate_xml_tree(root: Any) -> None: @@ -356,17 +404,7 @@ def _sl_validate_xml_tree(root: Any) -> None: def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: - wanted = {name.casefold() for name in names} - for child in list(element) if element is not None else []: - if _sl_local(child).casefold() not in wanted: - continue - text = (child.text or "").strip() if child.text else "" - if text: - return text - for key, value in child.attrib.items(): - if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: - return str(value).lstrip("#") - return default + return _sl_xml_scalar(element, tuple(names), default=default) def _sl_descendants(root: Any, *names: str) -> list[Any]: @@ -375,12 +413,32 @@ def _sl_descendants(root: Any, *names: str) -> list[Any]: def _sl_supplier(value: Any) -> str: - if isinstance(value, Mapping): - return _sl_safe_text(value.get("name"), "supplier") - if isinstance(value, list): - names = [_sl_supplier(item) for item in value] - return "; ".join(item for item in names if item) - return _sl_safe_text(value, "supplier") + """Normalize supplier forms and enforce the aggregate public bound.""" + if value is None: + return "" + if isinstance(value, str): + result = _sl_safe_text(value, "supplier", limit=512) + elif isinstance(value, Mapping): + if "name" not in value: + raise SBOMLineageError("supplier object requires a name") + result = _sl_safe_text(value.get("name"), "supplier", required=True, limit=512) + elif isinstance(value, list): + if len(value) > _SL_MAX_REFERENCES: + raise SBOMLineageError("supplier list exceeds its bound") + parts: list[str] = [] + for item in value: + if isinstance(item, (list, tuple, set)) or item is None: + raise SBOMLineageError("supplier list contains an invalid item") + part = _sl_supplier(item) + if not part: + raise SBOMLineageError("supplier list contains an empty item") + parts.append(part) + result = "; ".join(parts) + else: + raise SBOMLineageError("supplier must be a string, object, or list") + if len(result) > 512: + raise SBOMLineageError("supplier exceeds 512 characters") + return result def _sl_safe_text(value: Any, field: str, *, required: bool = False, limit: int = 512) -> str: @@ -482,6 +540,18 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di return result +def _sl_bind_component_alias(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: + existing = component_id_map.get(alias) + if existing is not None and existing != normalized_id: + raise SBOMLineageError("ambiguous component alias") + component_id_map[alias] = normalized_id + + +def _sl_bind_component_alias_variants(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: + for candidate in _sl_decoded_variants(alias): + _sl_bind_component_alias(component_id_map, candidate, normalized_id) + + def _sl_component( *, component_id: Any, @@ -497,13 +567,15 @@ def _sl_component( ) -> dict[str, Any]: normalized_name = _sl_safe_text(name, "component_name", required=True, limit=256) normalized_version = _sl_safe_text(version, "component_version", limit=128) + normalized_supplier = _sl_supplier(supplier) normalized_id = _sl_id(component_id, "component ID") - if isinstance(component_id, str) and component_id_map is not None: - component_id_map[component_id] = normalized_id + raw_component_id = component_id if isinstance(component_id, str) else "" if _sl_node_kind(normalized_id) != "component": normalized_id = "component:sha256:" + hashlib.sha256(normalized_id.encode("utf-8")).hexdigest() - if isinstance(component_id, str) and component_id_map is not None: - component_id_map[component_id] = normalized_id + if component_id_map is not None: + if raw_component_id and _sl_node_kind(raw_component_id) != "document": + _sl_bind_component_alias_variants(component_id_map, raw_component_id, normalized_id) + _sl_bind_component_alias(component_id_map, normalized_id, normalized_id) ids = {normalized_id} component_truncated: list[str] = list(truncated or []) if identifiers is not None and not isinstance(identifiers, (list, tuple, set)): @@ -517,7 +589,7 @@ def _sl_component( ids.add(text) if component_id_map is not None: for identifier in ids: - component_id_map.setdefault(identifier, normalized_id) + _sl_bind_component_alias(component_id_map, identifier, normalized_id) safe_references = [] if references is not None and not isinstance(references, (list, tuple)): raise SBOMLineageError("component references must be a list") @@ -553,16 +625,16 @@ def _sl_component( unknown = [] if not normalized_version: unknown.append("version") - if not _sl_supplier(supplier): + if not normalized_supplier: unknown.append("supplier") if component_truncated: unknown.append("truncated:" + ",".join(sorted(set(component_truncated)))) - coverage_state = "complete" if normalized_version and _sl_supplier(supplier) and not component_truncated else "partial" + coverage_state = "complete" if normalized_version and normalized_supplier and not component_truncated else "partial" return { "component_id": normalized_id, "name": normalized_name, "version": normalized_version or None, - "supplier": _sl_supplier(supplier) or None, + "supplier": normalized_supplier or None, "identifiers": sorted(ids), "references": sorted(safe_references, key=lambda item: (item.get("type", ""), item.get("locator", ""))), "licenses": sorted(set(license_values)), @@ -571,12 +643,16 @@ def _sl_component( } -def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None, component_id_map: Mapping[str, str] | None = None) -> dict[str, Any]: +def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None, component_id_map: Mapping[str, str] | None = None, reserved_ids: set[str] | None = None) -> dict[str, Any]: if component_id_map is not None: - if isinstance(source, str): - source = component_id_map.get(source, source) - if isinstance(target, str): - target = component_id_map.get(target, target) + if isinstance(source, str) and source in component_id_map: + if reserved_ids is not None and source in reserved_ids and component_id_map[source] != source: + raise SBOMLineageError("reserved relationship endpoint cannot bind to a component") + source = component_id_map[source] + if isinstance(target, str) and target in component_id_map: + if reserved_ids is not None and target in reserved_ids and component_id_map[target] != target: + raise SBOMLineageError("reserved relationship endpoint cannot bind to a component") + target = component_id_map[target] source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") @@ -643,7 +719,7 @@ def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = N component_id_map=component_id_map, ) if component_id_map is not None and isinstance(raw.get("purl"), str): - component_id_map.setdefault(raw["purl"], component["component_id"]) + _sl_bind_component_alias_variants(component_id_map, raw["purl"], component["component_id"]) return component @@ -651,14 +727,14 @@ def _sl_validate_spdx_version(value: Any) -> str: version_text = _sl_text(value, "SPDX version", required=True, limit=32) match = _SL_SPDX_VERSION_RE.fullmatch(version_text) if not match or match.group(1) not in _SL_SPDX_VERSIONS: - raise SBOMLineageError(f"unsupported SPDX version: {version_text}") + raise SBOMLineageError("unsupported SPDX version") return match.group(1) def _sl_validate_cdx_version(value: Any) -> str: version_text = _sl_text(value, "CycloneDX version", required=True, limit=32) if version_text not in _SL_CDX_VERSIONS: - raise SBOMLineageError(f"unsupported CycloneDX version: {version_text}") + raise SBOMLineageError("unsupported CycloneDX version") return version_text @@ -672,6 +748,8 @@ def _sl_spdx_document_id(value: Any) -> str: def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") + if _sl_node_kind(document_id) != "document": + raise SBOMLineageError("document ID must use the document namespace") if len(components) > _SL_MAX_COMPONENTS: raise SBOMLineageError("normalized SBOM components exceed their global bound") if len(relationships) > _SL_MAX_RELATIONSHIPS: @@ -683,11 +761,10 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu if not isinstance(component_id, str) or component_id in component_ids: raise SBOMLineageError("duplicate or missing component ID") component_ids.add(component_id) - if fmt == "SPDX" and document_id in component_ids: - raise SBOMLineageError("SPDX document ID collides with a component ID") + if document_id in component_ids: + raise SBOMLineageError("document ID collides with a component ID") known_ids = set(component_ids) - if fmt == "SPDX": - known_ids.add(document_id) + known_ids.add(document_id) dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) truncation = sorted(set(truncated or [])) unknown = [] @@ -753,7 +830,7 @@ def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: creators = creation.get("creators", []) if not isinstance(creators, list): raise SBOMLineageError("creationInfo.creators must be a list") - supplier = _sl_supplier(creators[0] if creators else "") + supplier = _sl_supplier(creators) packages = _sl_list(value.get("packages"), "packages") raw_relationships = _sl_list(value.get("relationships"), "relationships") truncated = [] @@ -774,6 +851,7 @@ def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: relationships.append(_sl_relationship( raw.get("spdxElementId"), raw.get("relatedSpdxElement"), raw.get("relationshipType", "related"), truncated=truncated, component_id_map=component_id_map, + reserved_ids={document_id}, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -814,6 +892,14 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s component_id_map: dict[str, str] = {} for raw in raw_components: components.append(_sl_cdx_component(raw, truncated=truncated, component_id_map=component_id_map)) + creators = metadata.get("authors", []) + if not isinstance(creators, list): + raise SBOMLineageError("metadata.authors must be a list") + supplier = _sl_supplier(creators if creators else (metadata_component or {})) + if "serialNumber" in value: + document_id = _sl_id(value.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() relationships = [] for raw in raw_dependency_list[:_SL_MAX_RELATIONSHIPS]: if not isinstance(raw, Mapping): @@ -830,15 +916,7 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s if len(targets) > allowed_targets: truncated.append("dependency_edges") for target in targets[:allowed_targets]: - relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated, component_id_map=component_id_map)) - creators = metadata.get("authors", []) - if not isinstance(creators, list): - raise SBOMLineageError("metadata.authors must be a list") - supplier = _sl_supplier(creators[0] if creators else (metadata_component or {})) - if "serialNumber" in value: - document_id = _sl_id(value.get("serialNumber"), "serialNumber") - else: - document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() + relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id})) return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_text(metadata.get("name"), "document_name"), created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, @@ -849,13 +927,18 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: documents = _sl_descendants(root, "SpdxDocument") + if len(documents) > 1: + raise SBOMLineageError("multiple RDF SpdxDocument nodes are not allowed") document = documents[0] if documents else root version = _sl_validate_spdx_version(_sl_xml_value(document, "spdxVersion")) raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") if not raw_document_id: raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") document_id = _sl_spdx_document_id(raw_document_id) - creation = next(iter(_sl_descendants(document, "creationInfo")), None) + creation_nodes = _sl_descendants(document, "creationInfo") + if len(creation_nodes) > 1: + raise SBOMLineageError("XML singleton element is duplicated") + creation = creation_nodes[0] if creation_nodes else None created_at = _sl_xml_value(creation, "created") creator = _sl_xml_value(creation, "creator") packages = _sl_descendants(document, "Package", "package") @@ -904,7 +987,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, _sl_xml_value(raw, "spdxElementId"), _sl_xml_value(raw, "relatedSpdxElement"), _sl_xml_value(raw, "relationshipType", default="related"), - truncated=truncated, component_id_map=component_id_map, + truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id}, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -916,7 +999,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) document_id = _sl_spdx_document_id(_sl_xml_text(root, "SPDXID")) - creation = _sl_child(root, "creationInfo") + creation = _sl_single_child(root, "creationInfo") created_at = _sl_xml_text(creation, "created") if creation is not None else "" creator = _sl_xml_text(creation, "creator") if creation is not None else "" packages = _sl_children(root, "package") @@ -957,7 +1040,7 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] relationships.append(_sl_relationship( _sl_xml_text(raw, "spdxElementId"), _sl_xml_text(raw, "relatedSpdxElement"), _sl_xml_text(raw, "relationshipType", default="related"), - truncated=truncated, component_id_map=component_id_map, + truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id}, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -973,7 +1056,7 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo purl = _sl_xml_text(raw, "purl") if purl: identifiers.append(_sl_safe_locator(purl, "purl")) - external = _sl_child(raw, "externalReferences") + external = _sl_single_child(raw, "externalReferences") if external is not None: reference_nodes = _sl_children(external, "reference") if len(reference_nodes) > _SL_MAX_REFERENCES: @@ -987,7 +1070,7 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo truncated=component_truncated, )) licenses = [] - license_container = _sl_child(raw, "licenses") + license_container = _sl_single_child(raw, "licenses") if license_container is not None: license_nodes = _sl_children(license_container, "license") if len(license_nodes) > _SL_MAX_LICENSES: @@ -998,12 +1081,12 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo truncated.extend(item for item in component_truncated if item not in truncated) component = _sl_component( component_id=raw.attrib.get("bom-ref"), name=_sl_xml_text(raw, "name"), - version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_child(raw, "supplier"), "name"), + version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_single_child(raw, "supplier"), "name"), identifiers=identifiers, references=refs, component_type=raw.attrib.get("type"), licenses=licenses, truncated=component_truncated, component_id_map=component_id_map, ) if component_id_map is not None and purl: - component_id_map.setdefault(purl, component["component_id"]) + _sl_bind_component_alias_variants(component_id_map, purl, component["component_id"]) return component @@ -1013,13 +1096,13 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if not match: raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") version = _sl_validate_cdx_version(match.group(1)) - metadata = _sl_child(root, "metadata") - metadata_component = _sl_child(metadata, "component") if metadata is not None else None + metadata = _sl_single_child(root, "metadata") + metadata_component = _sl_single_child(metadata, "component") if metadata is not None else None raw_components = [] truncated: list[str] = [] if metadata_component is not None: raw_components.append(metadata_component) - components_node = _sl_child(root, "components") + components_node = _sl_single_child(root, "components") if components_node is not None: component_nodes = _sl_children(components_node, "component") component_budget = _SL_MAX_COMPONENTS - (1 if metadata_component is not None else 0) @@ -1030,8 +1113,12 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, components = [] for raw in raw_components: components.append(_sl_cdx_xml_component(raw, truncated=truncated, component_id_map=component_id_map)) + if "serialNumber" in root.attrib: + document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() relationships = [] - dependencies_node = _sl_child(root, "dependencies") + dependencies_node = _sl_single_child(root, "dependencies") if dependencies_node is not None: dependency_nodes = _sl_children(dependencies_node, "dependency") if len(dependency_nodes) > _SL_MAX_RELATIONSHIPS: @@ -1049,14 +1136,11 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if len(children) > allowed_targets: truncated.append("dependency_edges") for child in children[:allowed_targets]: - relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated, component_id_map=component_id_map)) + relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id})) timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" - authors = _sl_child(metadata, "authors") if metadata is not None else None - author = _sl_xml_text(_sl_child(authors, "author"), "name") if authors is not None else "" - if "serialNumber" in root.attrib: - document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber") - else: - document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() + authors = _sl_single_child(metadata, "authors") if metadata is not None else None + author_nodes = _sl_children(authors, "author") if authors is not None else [] + author = _sl_supplier([_sl_xml_text(item, "name") for item in author_nodes]) if author_nodes else "" return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, @@ -1076,7 +1160,7 @@ def _sl_read_bounded(path: Path) -> bytes: except SBOMLineageError: raise except OSError as exc: - raise SBOMLineageError(f"could not open SBOM safely: {exc}") from exc + raise SBOMLineageError("could not open SBOM safely") from exc try: opened = _sl_os.fstat(fd) if not _sl_stat.S_ISREG(opened.st_mode): @@ -1099,7 +1183,7 @@ def _sl_read_bounded(path: Path) -> bytes: except SBOMLineageError: raise except OSError as exc: - raise SBOMLineageError(f"could not read SBOM safely: {exc}") from exc + raise SBOMLineageError("could not read SBOM safely") from exc finally: _sl_os.close(fd) @@ -1139,9 +1223,18 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) -def _sl_load_json_bounded(source: Path | bytes, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: +def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: """Load JSON through the shared byte and nesting bounds.""" - raw_bytes = _sl_read_bounded(source) if isinstance(source, Path) else bytes(source) + if isinstance(source, Path): + raw_bytes = _sl_read_bounded(source) + elif isinstance(source, bytes): + raw_bytes = source + elif isinstance(source, (bytearray, memoryview)): + raw_bytes = bytes(source) + else: + raise SBOMLineageError("bounded JSON input must be bytes or a regular file") + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") _sl_validate_json_depth(raw_bytes) try: value = json.loads( @@ -1198,6 +1291,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = document elif isinstance(document, bytearray): raw_bytes = bytes(document) + elif isinstance(document, memoryview): + raw_bytes = document.tobytes() elif isinstance(document, Mapping): try: raw_bytes = _sl_json(document).encode("utf-8") @@ -1240,9 +1335,13 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An value, raw_bytes = _sl_payload(document) normalized_source = _sl_safe_source_ref(source_ref, raw_bytes) if isinstance(value, Mapping): - if value.get("spdxVersion") is not None: + has_spdx_marker = "spdxVersion" in value + has_cdx_marker = "bomFormat" in value + if has_spdx_marker and has_cdx_marker: + raise SBOMLineageError("conflicting SBOM format markers") + if has_spdx_marker: return _sl_parse_spdx_json(value, raw_bytes, normalized_source) - if value.get("bomFormat") is not None: + if has_cdx_marker: return _sl_parse_cdx_json(value, raw_bytes, normalized_source) raise SBOMLineageError("SBOM format is missing or unsupported") root_name = _sl_local(value).casefold() @@ -1258,7 +1357,12 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An def _sl_node_kind(node_id: str) -> str: - if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("SPDXRef-DOCUMENT-") or node_id.startswith("document:"): + if ( + node_id == "SPDXRef-DOCUMENT" + or node_id.startswith("SPDXRef-DOCUMENT-") + or node_id.startswith("document:") + or node_id.startswith("urn:uuid:") + ): return "document" if node_id.startswith("artifact:"): return "artifact" @@ -1412,8 +1516,7 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") def _sl_expected_document_coverage(document_id: str, fmt: str, document_name: str | None, created_at: str | None, supplier: str | None, components: list[dict[str, Any]], relationships: list[dict[str, Any]], truncated: list[str]) -> dict[str, Any]: component_ids = {item["component_id"] for item in components} known_ids = set(component_ids) - if fmt == "SPDX": - known_ids.add(document_id) + known_ids.add(document_id) dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) unknown: list[str] = [] if not components: @@ -1475,8 +1578,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non document_id_text = _sl_strict_text(document.get("document_id"), "document_id", limit=256) assert document_id_text is not None document_id = _sl_id(document_id_text, "document_id") - if fmt == "SPDX" and _sl_node_kind(document_id) != "document": - raise SBOMLineageError("SPDX document ID must use the document namespace") + if _sl_node_kind(document_id) != "document": + raise SBOMLineageError("document ID must use the document namespace") for field, limit in (("document_name", 256), ("created_at", 128), ("supplier", 512)): _sl_strict_text(document.get(field), field, allow_none=True, limit=limit) document_sha = document.get("document_sha256") @@ -1497,8 +1600,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non raise SBOMLineageError("normalized SBOM contains duplicate component IDs") seen.add(checked["component_id"]) checked_components.append(checked) - if fmt == "SPDX" and document_id in seen: - raise SBOMLineageError("SPDX document ID collides with a component ID") + if document_id in seen: + raise SBOMLineageError("document ID collides with a component ID") checked_relationships = [_sl_validate_relationship(item) for item in relationships] coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") @@ -1532,7 +1635,7 @@ def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = try: checked = _sl_validate_document(document) if raw_document is None else _sl_rebind_document(document, raw_document) except (SBOMLineageError, TypeError, ValueError) as exc: - return {"valid": False, "error": str(exc)} + return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} @@ -1608,6 +1711,31 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any raise SBOMLineageError("raw_documents may only rebind normalized SBOM projections") normalized.append(ingest_sbom_document(document)) nodes: dict[str, dict[str, Any]] = {} + document_ids: set[str] = set() + component_ids_all = { + component["component_id"] + for document in normalized + for component in document.get("components", []) + } + for document in normalized: + document_id = document["document_id"] + if document_id in document_ids: + raise SBOMLineageError("duplicate document ID") + if document_id in component_ids_all: + raise SBOMLineageError("document ID collides with a component ID") + document_ids.add(document_id) + if len(nodes) >= _SL_MAX_NODES: + raise SBOMLineageError("lineage nodes exceed its global bound") + document_coverage = document["coverage"] + nodes[document_id] = { + "node_id": document_id, + "kind": "document", + "coverage": { + "state": document_coverage["state"], + "unknown": [] if document_coverage["state"] == "complete" else ["document_metadata"], + "truncated": document_coverage["truncated"], + }, + } component_fingerprints: dict[str, str] = {} native_edges: list[dict[str, Any]] = [] for document in normalized: @@ -1615,7 +1743,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any component_id = component["component_id"] fingerprint = _sl_sha(component) if component_id in component_fingerprints and component_fingerprints[component_id] != fingerprint: - raise SBOMLineageError(f"conflicting duplicate component ID: {component_id}") + raise SBOMLineageError("conflicting duplicate component ID") if component_id in component_fingerprints: continue if len(nodes) >= _SL_MAX_NODES: @@ -1663,19 +1791,28 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any return body -def _sl_validate_node_coverage(value: Any, *, component: bool) -> dict[str, Any]: +def _sl_validate_node_coverage(value: Any, *, component: bool, document: bool = False, expected: Mapping[str, Any] | None = None) -> dict[str, Any]: _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "node.coverage") state = value.get("state") if state not in _SL_COVERAGE: raise SBOMLineageError("node.coverage.state is invalid") unknown = _sl_string_list(value.get("unknown"), "node.coverage.unknown", maximum=32) truncated = _sl_string_list(value.get("truncated"), "node.coverage.truncated", maximum=32) - if not component and (state != "partial" or unknown != ["node_metadata"] or truncated): + if document: + if expected is None or {"state", "unknown", "truncated"} != set(expected): + raise SBOMLineageError("document node coverage binding is invalid") + if {"state": state, "unknown": unknown, "truncated": truncated} != dict(expected): + raise SBOMLineageError("document node coverage is not bound to its document") + elif not component and (state != "partial" or unknown != ["node_metadata"] or truncated): raise SBOMLineageError("external lineage node coverage is invalid") return {"state": state, "unknown": unknown, "truncated": truncated} -def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[str, Mapping[str, Any]]]]) -> str: +def _sl_validate_lineage_node( + node: Any, + component_map: Mapping[str, list[tuple[str, Mapping[str, Any]]]], + document_map: Mapping[str, Mapping[str, Any]], +) -> str: if not isinstance(node, Mapping): raise SBOMLineageError("lineage node must be an object") node_id = _sl_strict_text(node.get("node_id"), "node.node_id", limit=256) @@ -1685,6 +1822,19 @@ def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[ expected_kind = _sl_node_kind(node_id) if kind != expected_kind: raise SBOMLineageError("lineage node kind does not match its ID") + if kind == "document": + _sl_require_keys(node, {"node_id", "kind", "coverage"}, set(), "document node") + expected_document = document_map.get(node_id) + if expected_document is None: + raise SBOMLineageError("document node is not bound to a source document") + document_coverage = expected_document["coverage"] + expected_node_coverage = { + "state": document_coverage["state"], + "unknown": [] if document_coverage["state"] == "complete" else ["document_metadata"], + "truncated": document_coverage["truncated"], + } + _sl_validate_node_coverage(node.get("coverage"), component=False, document=True, expected=expected_node_coverage) + return node_id component_fields = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} if kind == "component" and component_fields.issubset(set(node)): required = component_fields | {"node_id", "kind", "document_sha256"} @@ -1736,22 +1886,36 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match lineage documents within its bound") checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] + document_map: dict[str, Mapping[str, Any]] = {} component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} + component_ids_all: set[str] = set() for document in checked_documents: + document_id = document["document_id"] + if document_id in document_map: + raise SBOMLineageError("duplicate document ID") + document_map[document_id] = document for component in document["components"]: + component_ids_all.add(component["component_id"]) component_map.setdefault(component["component_id"], []).append((document["document_sha256"], component)) + if set(document_map).intersection(component_ids_all): + raise SBOMLineageError("document ID collides with a component ID") for component_id, candidates in component_map.items(): if len({_sl_sha(item) for _, item in candidates}) > 1: - raise SBOMLineageError(f"conflicting component projection for {component_id}") + raise SBOMLineageError("conflicting component projection") node_ids: set[str] = set() full_component_ids: set[str] = set() + full_document_ids: set[str] = set() for node in nodes: - node_id = _sl_validate_lineage_node(node, component_map) + node_id = _sl_validate_lineage_node(node, component_map, document_map) if node_id in node_ids: raise SBOMLineageError("lineage contains duplicate nodes") node_ids.add(node_id) + if isinstance(node, Mapping) and node.get("kind") == "document": + full_document_ids.add(node_id) if isinstance(node, Mapping) and "component_id" in node: full_component_ids.add(node_id) + if full_document_ids != set(document_map): + raise SBOMLineageError("lineage document nodes are not bound to all documents") if full_component_ids != set(component_map): raise SBOMLineageError("lineage component nodes are not bound to all documents") checked_edges = [] @@ -1789,7 +1953,7 @@ def verify_sbom_lineage(lineage: Mapping[str, Any], raw_documents: Any | None = try: loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: - return {"valid": False, "error": str(exc)} + return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -1967,7 +2131,13 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin authoritative_lineage = json.loads(authority[0]) except (TypeError, json.JSONDecodeError) as exc: raise SBOMLineageError("authoritative lineage receipt is malformed") from exc + assert authoritative_lineage is not None authority_node_ids: set[str] = set() + authority_document_map = { + document.get("document_id"): document + for document in authoritative_lineage.get("documents", []) + if isinstance(document, Mapping) and isinstance(document.get("document_id"), str) + } for node in lineage_nodes: if not isinstance(node, Mapping): raise SBOMLineageError("query authoritative node must be an object") @@ -1979,6 +2149,20 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin if kind != _sl_node_kind(node_id) or node_id in authority_node_ids: raise SBOMLineageError("query authoritative node identity is invalid") authority_node_ids.add(node_id) + if kind == "document": + document = authority_document_map.get(node_id) + if document is None: + raise SBOMLineageError("query document node is not bound to authoritative documents") + document_coverage = document.get("coverage") + if not isinstance(document_coverage, Mapping): + raise SBOMLineageError("query document node coverage is invalid") + expected_node_coverage = { + "state": document_coverage.get("state"), + "unknown": [] if document_coverage.get("state") == "complete" else ["document_metadata"], + "truncated": document_coverage.get("truncated"), + } + _sl_validate_node_coverage(node.get("coverage"), component=False, document=True, expected=expected_node_coverage) + continue full_component = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} if full_component.issubset(set(node)): _sl_validate_component({key: node[key] for key in full_component}) @@ -2095,7 +2279,7 @@ def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lin try: checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: - return {"valid": False, "error": str(exc)} + return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} @@ -2125,6 +2309,7 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: output = getattr(args, "output", None) if command == "ingest": document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) + _sl_validate_document(document) elif command == "merge": document_paths = _sl_cli_bounded_paths(getattr(args, "documents", None), "documents", required=True) raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") @@ -2147,6 +2332,7 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: else: edges = [] document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) + _sl_loaded_lineage(document, raw_documents=raw_inputs) elif command == "query": raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage, _ = _sl_load_json_bounded(Path(args.lineage)) @@ -2161,6 +2347,7 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: document = query_sbom_lineage( lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, ) + _sl_validate_query_result(document, lineage, raw_inputs) else: raise SBOMLineageError("command must be ingest, merge, or query") serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" @@ -2172,9 +2359,24 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: digest_key = "lineage_digest" if "lineage_digest" in document else "query_digest" if "query_digest" in document else "ingestion_digest" print(f"sbom {command} -> {output}\n{digest_key}: {document[digest_key]}") return 0 - except (OSError, TypeError, ValueError, SBOMLineageError) as exc: + except SBOMLineageError as exc: + error = str(exc)[:160] + if getattr(args, "json", False): + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": error, "valid": False}, sort_keys=True)) + else: + print(f"sbom: {error}") + return 1 + except OSError: + error = "SBOM filesystem operation failed" + if getattr(args, "json", False): + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": error, "valid": False}, sort_keys=True)) + else: + print(f"sbom: {error}") + return 1 + except (TypeError, ValueError): + error = "SBOM input is invalid" if getattr(args, "json", False): - print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": str(exc), "valid": False}, sort_keys=True)) + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": error, "valid": False}, sort_keys=True)) else: - print(f"sbom: {exc}") + print(f"sbom: {error}") return 1 diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 47b81b61..7c264a88 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -187,6 +187,7 @@ def test_untrusted_normalized_documents_and_conflicting_duplicate_ids_fail_close with pytest.raises(perseus.SBOMLineageError, match="digest"): perseus.build_sbom_lineage([tampered]) alternate = json.loads(_load("spdx-app.json")) + alternate["SPDXID"] = "SPDXRef-DOCUMENT-ALTERNATE" alternate["packages"][0]["name"] = "different-component" alternate_document = perseus.ingest_sbom_document(alternate, source_ref="artifact:alternate") with pytest.raises(perseus.SBOMLineageError, match="duplicate component ID"): @@ -348,7 +349,13 @@ def test_all_collection_caps_are_recorded_and_downgrade_coverage(): assert "externalRefs" in document["coverage"]["truncated"] assert document["coverage"]["state"] == "partial" - lineage = perseus.build_sbom_lineage([document] * 65, edges=[]) + documents = [] + for index in range(65): + unique_payload = json.loads(json.dumps(payload)) + unique_payload["SPDXID"] = f"SPDXRef-DOCUMENT-{index}" + unique_payload["relationships"][0]["spdxElementId"] = f"SPDXRef-DOCUMENT-{index}" + documents.append(perseus.ingest_sbom_document(unique_payload, source_ref=f"artifact:cap-{index}")) + lineage = perseus.build_sbom_lineage(documents, edges=[]) assert "documents" in lineage["coverage"]["truncated"] assert lineage["coverage"]["state"] != "complete" @@ -1061,3 +1068,193 @@ def test_duplicate_json_object_keys_are_rejected_before_projection(): raw = b'{"bomFormat":"CycloneDX","bomFormat":"SPDX","specVersion":"1.5"}' with pytest.raises(perseus.SBOMLineageError, match="duplicate"): perseus.ingest_sbom_document(raw) + + +def test_percent_decoded_git_style_userinfo_is_not_persisted(): + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["purl"] = "pkg:generic/example@1?repository_url=git%3Aalice%3Apw%40host%3Arepo" + component["externalReferences"] = [{ + "type": "website", + "url": "git%3Aalice%3Apw%40host%3Arepo", + }] + document = perseus.ingest_sbom_document(payload, source_ref="build:alice%3Apw%40host:repo") + serialized = json.dumps(document, sort_keys=True) + for marker in ("alice", "pw", "host", "repo"): + assert marker not in serialized + assert document["source_ref"].startswith("sha256:") + + +def test_public_errors_do_not_echo_untrusted_version_or_filesystem_exception(tmp_path, monkeypatch): + payload = json.loads(_load("spdx-app.json")) + payload["spdxVersion"] = "SPDX-9.9-UNTRUSTED_VERSION" + with pytest.raises(perseus.SBOMLineageError) as version_error: + perseus.ingest_sbom_document(payload) + assert "UNTRUSTED_VERSION" not in str(version_error.value) + + path = tmp_path / "input.json" + path.write_bytes(b"{}") + monkeypatch.setattr(perseus._sl_os, "open", lambda *args, **kwargs: (_ for _ in ()).throw(OSError("UNTRUSTED_FILESYSTEM_DETAIL"))) + with pytest.raises(perseus.SBOMLineageError) as filesystem_error: + perseus._sl_read_bounded(path) + assert "UNTRUSTED_FILESYSTEM_DETAIL" not in str(filesystem_error.value) + assert str(filesystem_error.value) == "could not open SBOM safely" + + +def test_cli_error_serialization_does_not_echo_paths_or_untrusted_version(tmp_path, capsys): + missing_args = type("Args", (), { + "sbom_command": "ingest", "document": str(tmp_path / "PATH_SECRET_INPUT.json"), + "source_ref": "", "output": None, "json": True, + })() + assert perseus.cmd_sbom(missing_args, {}) == 1 + missing_output = capsys.readouterr().out + assert "PATH_SECRET_INPUT" not in missing_output + assert "filesystem" in missing_output or "open" in missing_output + + payload = json.loads(_load("spdx-app.json")) + payload["spdxVersion"] = "SPDX-9.9-CLI_UNTRUSTED_VERSION" + source = tmp_path / "bad-version.json" + source.write_text(json.dumps(payload), encoding="utf-8") + version_args = type("Args", (), { + "sbom_command": "ingest", "document": str(source), "source_ref": "", + "output": None, "json": True, + })() + assert perseus.cmd_sbom(version_args, {}) == 1 + version_output = capsys.readouterr().out + assert "CLI_UNTRUSTED_VERSION" not in version_output + assert "version" in version_output + + +def test_load_json_bounded_caps_all_in_memory_byte_buffers_and_rejects_text(): + oversized = b"{" + b" " * perseus._SL_MAX_INPUT_BYTES + b"}" + for source in (oversized, bytearray(oversized), memoryview(oversized)): + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_load_json_bounded(source) + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_load_json_bounded("{}") + + +def test_conflicting_json_format_discriminators_are_rejected(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["spdxVersion"] = "SPDX-2.3" + payload["SPDXID"] = "SPDXRef-DOCUMENT" + with pytest.raises(perseus.SBOMLineageError, match="format"): + perseus.ingest_sbom_document(payload) + + +def test_xml_singleton_conflicts_and_multiple_rdf_documents_fail(): + xml = _load("spdx-app.xml").decode("utf-8") + conflicting = xml.replace( + "SPDX-2.3", + "SPDX-2.3SPDX-2.2", + ) + with pytest.raises(perseus.SBOMLineageError, match="singleton|conflict|version"): + perseus.ingest_sbom_document(conflicting) + + rdf = _load("spdx-rdf.xml").decode("utf-8") + start = rdf.index(" ") + len(" ") + node = rdf[start:end] + multiple = rdf.replace("", node + "\n") + with pytest.raises(perseus.SBOMLineageError, match="SpdxDocument|RDF|multiple"): + perseus.ingest_sbom_document(multiple) + + +def test_distinct_component_refs_with_one_purl_are_rejected_as_ambiguous(): + payload = json.loads(_load("cyclonedx-app.json")) + first = payload["components"][0] + shared_purl = "pkg:generic/shared-component@1" + first["purl"] = shared_purl + first["bom-ref"] = "pkg:generic/first-component@1" + second = json.loads(json.dumps(first)) + second["bom-ref"] = "pkg:generic/second-component@1" + second["name"] = "second-component" + payload["components"].append(second) + with pytest.raises(perseus.SBOMLineageError, match="ambiguous|alias|identifier"): + perseus.ingest_sbom_document(payload) + + +def test_supplier_shape_and_aggregate_output_are_bounded_at_ingestion(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["supplier"] = ["A" * 300, "B" * 300] + with pytest.raises(perseus.SBOMLineageError, match="supplier|bound"): + perseus.ingest_sbom_document(payload) + + malformed = json.loads(_load("cyclonedx-app.json")) + malformed["components"][0]["supplier"] = {"name": 123} + with pytest.raises(perseus.SBOMLineageError, match="supplier|string"): + perseus.ingest_sbom_document(malformed) + + +def test_cli_ingest_validates_normalized_document_before_success(tmp_path, capsys): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["supplier"] = ["A" * 300, "B" * 300] + source = tmp_path / "supplier.json" + output = tmp_path / "normalized.json" + source.write_text(json.dumps(payload), encoding="utf-8") + args = type("Args", (), { + "sbom_command": "ingest", "document": str(source), "source_ref": "", + "output": str(output), "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 1 + assert not output.exists() + failure = json.loads(capsys.readouterr().out) + assert failure["valid"] is False + assert "supplier" in failure["error"] + + +def test_cli_ingest_rejects_a_malformed_normalized_return_before_writing(tmp_path, monkeypatch, capsys): + valid = perseus.ingest_sbom_document(_load("cyclonedx-app.json"), source_ref="artifact:cli-validation") + invalid = json.loads(json.dumps(valid)) + invalid["components"][0]["supplier"] = "A" * 513 + monkeypatch.setattr(perseus, "ingest_sbom_document", lambda *args, **kwargs: invalid) + output = tmp_path / "invalid-normalized.json" + args = type("Args", (), { + "sbom_command": "ingest", "document": "ignored-input", "source_ref": "", + "output": str(output), "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 1 + assert not output.exists() + failure = json.loads(capsys.readouterr().out) + assert failure["valid"] is False + assert "digest" in failure["error"] or "bound" in failure["error"] + + +def test_duplicate_document_ids_and_document_component_collisions_fail_closed(): + first = perseus.ingest_sbom_document(_load("cyclonedx-app.json"), source_ref="artifact:first") + second = perseus.ingest_sbom_document(_load("cyclonedx-app.json"), source_ref="artifact:second") + with pytest.raises(perseus.SBOMLineageError, match="document"): + perseus.build_sbom_lineage([first, second], edges=[]) + + collision_payload = json.loads(_load("cyclonedx-app.json")) + collision_payload["serialNumber"] = collision_payload["components"][0]["bom-ref"] + with pytest.raises(perseus.SBOMLineageError, match="document|component|collision"): + perseus.ingest_sbom_document(collision_payload) + + +def test_known_spdx_document_nodes_preserve_complete_native_relationship_coverage(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:document-node") + assert document["coverage"]["state"] == "complete" + lineage = perseus.build_sbom_lineage([document], edges=[]) + document_node = next(node for node in lineage["nodes"] if node["node_id"] == document["document_id"]) + assert document_node["kind"] == "document" + assert document_node["coverage"]["state"] == "complete" + assert lineage["coverage"]["state"] == "complete" + assert perseus.verify_sbom_lineage(lineage)["valid"] is True + + +def test_known_cyclonedx_document_endpoint_remains_bound_and_complete(): + payload = json.loads(_load("cyclonedx-app.json")) + document_id = "urn:uuid:document-level-native" + payload["serialNumber"] = document_id + payload["metadata"]["name"] = "document-level-native" + component_ref = payload["components"][0]["purl"] + payload["dependencies"] = [{"ref": document_id, "dependsOn": [component_ref]}] + document = perseus.ingest_sbom_document(payload, source_ref="artifact:cdx-document-node") + assert document["coverage"]["state"] == "complete" + assert document["relationships"][0]["from"] == document_id + lineage = perseus.build_sbom_lineage([document], edges=[]) + assert lineage["coverage"]["state"] == "complete" + node = next(item for item in lineage["nodes"] if item["node_id"] == document_id) + assert node["kind"] == "document" + assert perseus.verify_sbom_lineage(lineage)["valid"] is True From fcd27738b6941cf247514204fa9bde914eb58150 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 21:08:05 +0000 Subject: [PATCH 10/40] build(sbom): refresh generated artifact --- perseus.py | 436 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 319 insertions(+), 117 deletions(-) diff --git a/perseus.py b/perseus.py index bb97dcfd..a17b0176 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "11b06fa" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "8d839d0" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44469,8 +44469,9 @@ def cmd_code_map(args, cfg) -> int: _SL_SPDX_VERSION_RE = re.compile(r"^SPDX-(\d+\.\d+)$") _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") -_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=)") +_SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=|authorization\s*[/=:])") _SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw)(?:[/?:=&_.-]|$)") +_SL_PRIVACY_MARKER_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(api[_-]?key|authorization|password|passwd|secret|token|credential)(?:[/?:=&_.-]|$)") _SL_PUBLIC_PURL_QUERY_KEYS = frozenset({"classifier", "extension", "type", "repository_url"}) _SL_REFERENCE_TYPES = frozenset({ "advisory", "attestation", "cve", "distribution", "documentation", "license", @@ -44530,6 +44531,13 @@ class SBOMLineageError(ValueError): """Raised when an SBOM or lineage projection cannot be verified.""" +def _sl_public_error(exc: BaseException) -> str: + """Serialize only bounded, non-exception-derived public error text.""" + if isinstance(exc, SBOMLineageError): + return str(exc)[:160] + return "SBOM input is invalid" + + def _sl_json(value: Any) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False) @@ -44544,7 +44552,7 @@ def _sl_json_object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: def _sl_reject_json_constant(value: str) -> None: - raise SBOMLineageError(f"non-finite JSON constant is not allowed: {value}") + raise SBOMLineageError("non-finite JSON constant is not allowed") def _sl_sha(value: Any) -> str: @@ -44556,19 +44564,47 @@ def _sl_ingestion_sha(unsigned: Mapping[str, Any], raw_bytes: bytes | None = Non return _sl_sha({"domain": "perseus-sbom-ingestion", "raw_sha256": raw_digest, "projection": unsigned}) -def _sl_private_locator(value: str) -> bool: - """Identify private/credential-bearing URI and identifier shapes.""" +def _sl_decoded_variants(value: str) -> list[str]: + """Return bounded, recursively percent-decoded inspection candidates.""" + candidates = [value] decoded = value for _ in range(min(len(value) + 1, 1025)): next_decoded = unquote(decoded) if next_decoded == decoded: break + candidates.append(next_decoded) decoded = next_decoded + return candidates + + +def _sl_userinfo_locator(value: str) -> bool: + """Detect URI and git-style userinfo without treating version ``@`` as auth.""" + for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): + if "@" in match.group(1): + return True + # ``git:user@host:repo`` and ``build:user@host/repo`` do not have ``://``. + # A versioned PURL/artifact ID ends in ``@version`` and has no locator-like + # suffix after the at-sign, so it remains a public identifier. + for at in (index for index, char in enumerate(value) if char == "@"): + suffix = value[at + 1:].split("#", 1)[0].split("?", 1)[0] + if "->" in suffix: + continue + if re.match(r"^[^/]+:[^/]+", suffix) or "/" in suffix: + return True + return False + + +def _sl_private_locator(value: str) -> bool: + """Identify private, credential-bearing, or non-public locator shapes.""" + decoded = _sl_decoded_variants(value)[-1] lowered = decoded.casefold() - if "#" in decoded or lowered.startswith("file:") or _SL_PRIVATE_LOCATOR_RE.search(decoded): - return True - authority = decoded.split("://", 1)[1].split("/", 1)[0] if "://" in decoded else "" - if authority and "@" in authority: + if ( + "#" in decoded + or lowered.startswith("file:") + or _SL_PRIVATE_LOCATOR_RE.search(decoded) + or _SL_PRIVACY_MARKER_RE.search(decoded) + or _sl_userinfo_locator(decoded) + ): return True if "?" in decoded: query = decoded.split("?", 1)[1].split("#", 1)[0] @@ -44580,22 +44616,12 @@ def _sl_private_locator(value: str) -> bool: def _sl_sensitive(value: str) -> bool: - """Return whether a scalar looks like it carries a credential.""" - candidates = [value] - decoded = value - # Decode until stable, bounded by the scalar length. Each successful - # percent-decoding pass must consume at least one encoded triplet, so this - # remains bounded for the 1024-character input limit. - for _ in range(min(len(value) + 1, 1025)): - next_decoded = unquote(decoded) - if next_decoded == decoded: - break - candidates.append(next_decoded) - decoded = next_decoded - for candidate in candidates: + """Return whether a scalar looks like it carries private material.""" + for candidate in _sl_decoded_variants(value): normalized = re.sub(r"[^a-z0-9]+", "_", candidate.casefold()) if ( _SL_SENSITIVE_REFERENCE_RE.search(candidate) + or _SL_PRIVACY_MARKER_RE.search(candidate) or _sl_private_locator(candidate) or any(marker in normalized for marker in _SL_FORBIDDEN_MARKERS) ): @@ -44771,11 +44797,33 @@ def _sl_children(element: Any, name: str) -> list[Any]: return [child for child in list(element) if _sl_local(child) == name] +def _sl_single_child(element: Any, name: str) -> Any | None: + matches = _sl_children(element, name) + if len(matches) > 1: + raise SBOMLineageError("XML singleton element is duplicated") + return matches[0] if matches else None + + +def _sl_xml_scalar(element: Any, names: tuple[str, ...], *, default: str = "") -> str: + wanted = {name.casefold() for name in names} + values: list[str] = [] + for child in list(element) if element is not None else []: + if _sl_local(child).casefold() not in wanted: + continue + text = (child.text or "").strip() if child.text else "" + scalar = text + for key, value in child.attrib.items(): + if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: + scalar = str(value).lstrip("#") + break + values.append(scalar) + if values and any(value != values[0] for value in values[1:]): + raise SBOMLineageError("XML singleton values conflict") + return values[0] if values else default + + def _sl_xml_text(element: Any, name: str, *, default: str = "") -> str: - if element is None: - return default - child = _sl_child(element, name) - return (child.text or "").strip() if child is not None and child.text else default + return _sl_xml_scalar(element, (name,), default=default) def _sl_validate_xml_tree(root: Any) -> None: @@ -44792,17 +44840,7 @@ def _sl_validate_xml_tree(root: Any) -> None: def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: - wanted = {name.casefold() for name in names} - for child in list(element) if element is not None else []: - if _sl_local(child).casefold() not in wanted: - continue - text = (child.text or "").strip() if child.text else "" - if text: - return text - for key, value in child.attrib.items(): - if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: - return str(value).lstrip("#") - return default + return _sl_xml_scalar(element, tuple(names), default=default) def _sl_descendants(root: Any, *names: str) -> list[Any]: @@ -44811,12 +44849,32 @@ def _sl_descendants(root: Any, *names: str) -> list[Any]: def _sl_supplier(value: Any) -> str: - if isinstance(value, Mapping): - return _sl_safe_text(value.get("name"), "supplier") - if isinstance(value, list): - names = [_sl_supplier(item) for item in value] - return "; ".join(item for item in names if item) - return _sl_safe_text(value, "supplier") + """Normalize supplier forms and enforce the aggregate public bound.""" + if value is None: + return "" + if isinstance(value, str): + result = _sl_safe_text(value, "supplier", limit=512) + elif isinstance(value, Mapping): + if "name" not in value: + raise SBOMLineageError("supplier object requires a name") + result = _sl_safe_text(value.get("name"), "supplier", required=True, limit=512) + elif isinstance(value, list): + if len(value) > _SL_MAX_REFERENCES: + raise SBOMLineageError("supplier list exceeds its bound") + parts: list[str] = [] + for item in value: + if isinstance(item, (list, tuple, set)) or item is None: + raise SBOMLineageError("supplier list contains an invalid item") + part = _sl_supplier(item) + if not part: + raise SBOMLineageError("supplier list contains an empty item") + parts.append(part) + result = "; ".join(parts) + else: + raise SBOMLineageError("supplier must be a string, object, or list") + if len(result) > 512: + raise SBOMLineageError("supplier exceeds 512 characters") + return result def _sl_safe_text(value: Any, field: str, *, required: bool = False, limit: int = 512) -> str: @@ -44918,6 +44976,18 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di return result +def _sl_bind_component_alias(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: + existing = component_id_map.get(alias) + if existing is not None and existing != normalized_id: + raise SBOMLineageError("ambiguous component alias") + component_id_map[alias] = normalized_id + + +def _sl_bind_component_alias_variants(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: + for candidate in _sl_decoded_variants(alias): + _sl_bind_component_alias(component_id_map, candidate, normalized_id) + + def _sl_component( *, component_id: Any, @@ -44933,13 +45003,15 @@ def _sl_component( ) -> dict[str, Any]: normalized_name = _sl_safe_text(name, "component_name", required=True, limit=256) normalized_version = _sl_safe_text(version, "component_version", limit=128) + normalized_supplier = _sl_supplier(supplier) normalized_id = _sl_id(component_id, "component ID") - if isinstance(component_id, str) and component_id_map is not None: - component_id_map[component_id] = normalized_id + raw_component_id = component_id if isinstance(component_id, str) else "" if _sl_node_kind(normalized_id) != "component": normalized_id = "component:sha256:" + hashlib.sha256(normalized_id.encode("utf-8")).hexdigest() - if isinstance(component_id, str) and component_id_map is not None: - component_id_map[component_id] = normalized_id + if component_id_map is not None: + if raw_component_id and _sl_node_kind(raw_component_id) != "document": + _sl_bind_component_alias_variants(component_id_map, raw_component_id, normalized_id) + _sl_bind_component_alias(component_id_map, normalized_id, normalized_id) ids = {normalized_id} component_truncated: list[str] = list(truncated or []) if identifiers is not None and not isinstance(identifiers, (list, tuple, set)): @@ -44953,7 +45025,7 @@ def _sl_component( ids.add(text) if component_id_map is not None: for identifier in ids: - component_id_map.setdefault(identifier, normalized_id) + _sl_bind_component_alias(component_id_map, identifier, normalized_id) safe_references = [] if references is not None and not isinstance(references, (list, tuple)): raise SBOMLineageError("component references must be a list") @@ -44989,16 +45061,16 @@ def _sl_component( unknown = [] if not normalized_version: unknown.append("version") - if not _sl_supplier(supplier): + if not normalized_supplier: unknown.append("supplier") if component_truncated: unknown.append("truncated:" + ",".join(sorted(set(component_truncated)))) - coverage_state = "complete" if normalized_version and _sl_supplier(supplier) and not component_truncated else "partial" + coverage_state = "complete" if normalized_version and normalized_supplier and not component_truncated else "partial" return { "component_id": normalized_id, "name": normalized_name, "version": normalized_version or None, - "supplier": _sl_supplier(supplier) or None, + "supplier": normalized_supplier or None, "identifiers": sorted(ids), "references": sorted(safe_references, key=lambda item: (item.get("type", ""), item.get("locator", ""))), "licenses": sorted(set(license_values)), @@ -45007,12 +45079,16 @@ def _sl_component( } -def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None, component_id_map: Mapping[str, str] | None = None) -> dict[str, Any]: +def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confidence: str = "high", coverage: str = "complete", evidence_refs: Any = None, truncated: list[str] | None = None, component_id_map: Mapping[str, str] | None = None, reserved_ids: set[str] | None = None) -> dict[str, Any]: if component_id_map is not None: - if isinstance(source, str): - source = component_id_map.get(source, source) - if isinstance(target, str): - target = component_id_map.get(target, target) + if isinstance(source, str) and source in component_id_map: + if reserved_ids is not None and source in reserved_ids and component_id_map[source] != source: + raise SBOMLineageError("reserved relationship endpoint cannot bind to a component") + source = component_id_map[source] + if isinstance(target, str) and target in component_id_map: + if reserved_ids is not None and target in reserved_ids and component_id_map[target] != target: + raise SBOMLineageError("reserved relationship endpoint cannot bind to a component") + target = component_id_map[target] source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") @@ -45079,7 +45155,7 @@ def _sl_cdx_component(raw: Mapping[str, Any], *, truncated: list[str] | None = N component_id_map=component_id_map, ) if component_id_map is not None and isinstance(raw.get("purl"), str): - component_id_map.setdefault(raw["purl"], component["component_id"]) + _sl_bind_component_alias_variants(component_id_map, raw["purl"], component["component_id"]) return component @@ -45087,14 +45163,14 @@ def _sl_validate_spdx_version(value: Any) -> str: version_text = _sl_text(value, "SPDX version", required=True, limit=32) match = _SL_SPDX_VERSION_RE.fullmatch(version_text) if not match or match.group(1) not in _SL_SPDX_VERSIONS: - raise SBOMLineageError(f"unsupported SPDX version: {version_text}") + raise SBOMLineageError("unsupported SPDX version") return match.group(1) def _sl_validate_cdx_version(value: Any) -> str: version_text = _sl_text(value, "CycloneDX version", required=True, limit=32) if version_text not in _SL_CDX_VERSIONS: - raise SBOMLineageError(f"unsupported CycloneDX version: {version_text}") + raise SBOMLineageError("unsupported CycloneDX version") return version_text @@ -45108,6 +45184,8 @@ def _sl_spdx_document_id(value: Any) -> str: def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") + if _sl_node_kind(document_id) != "document": + raise SBOMLineageError("document ID must use the document namespace") if len(components) > _SL_MAX_COMPONENTS: raise SBOMLineageError("normalized SBOM components exceed their global bound") if len(relationships) > _SL_MAX_RELATIONSHIPS: @@ -45119,11 +45197,10 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu if not isinstance(component_id, str) or component_id in component_ids: raise SBOMLineageError("duplicate or missing component ID") component_ids.add(component_id) - if fmt == "SPDX" and document_id in component_ids: - raise SBOMLineageError("SPDX document ID collides with a component ID") + if document_id in component_ids: + raise SBOMLineageError("document ID collides with a component ID") known_ids = set(component_ids) - if fmt == "SPDX": - known_ids.add(document_id) + known_ids.add(document_id) dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) truncation = sorted(set(truncated or [])) unknown = [] @@ -45189,7 +45266,7 @@ def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: creators = creation.get("creators", []) if not isinstance(creators, list): raise SBOMLineageError("creationInfo.creators must be a list") - supplier = _sl_supplier(creators[0] if creators else "") + supplier = _sl_supplier(creators) packages = _sl_list(value.get("packages"), "packages") raw_relationships = _sl_list(value.get("relationships"), "relationships") truncated = [] @@ -45210,6 +45287,7 @@ def _sl_parse_spdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: relationships.append(_sl_relationship( raw.get("spdxElementId"), raw.get("relatedSpdxElement"), raw.get("relationshipType", "related"), truncated=truncated, component_id_map=component_id_map, + reserved_ids={document_id}, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -45250,6 +45328,14 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s component_id_map: dict[str, str] = {} for raw in raw_components: components.append(_sl_cdx_component(raw, truncated=truncated, component_id_map=component_id_map)) + creators = metadata.get("authors", []) + if not isinstance(creators, list): + raise SBOMLineageError("metadata.authors must be a list") + supplier = _sl_supplier(creators if creators else (metadata_component or {})) + if "serialNumber" in value: + document_id = _sl_id(value.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() relationships = [] for raw in raw_dependency_list[:_SL_MAX_RELATIONSHIPS]: if not isinstance(raw, Mapping): @@ -45266,15 +45352,7 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s if len(targets) > allowed_targets: truncated.append("dependency_edges") for target in targets[:allowed_targets]: - relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated, component_id_map=component_id_map)) - creators = metadata.get("authors", []) - if not isinstance(creators, list): - raise SBOMLineageError("metadata.authors must be a list") - supplier = _sl_supplier(creators[0] if creators else (metadata_component or {})) - if "serialNumber" in value: - document_id = _sl_id(value.get("serialNumber"), "serialNumber") - else: - document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() + relationships.append(_sl_relationship(source, target, "depends_on", truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id})) return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_text(metadata.get("name"), "document_name"), created_at=_sl_text(metadata.get("timestamp"), "created_at"), supplier=supplier, @@ -45285,13 +45363,18 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: documents = _sl_descendants(root, "SpdxDocument") + if len(documents) > 1: + raise SBOMLineageError("multiple RDF SpdxDocument nodes are not allowed") document = documents[0] if documents else root version = _sl_validate_spdx_version(_sl_xml_value(document, "spdxVersion")) raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") if not raw_document_id: raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") document_id = _sl_spdx_document_id(raw_document_id) - creation = next(iter(_sl_descendants(document, "creationInfo")), None) + creation_nodes = _sl_descendants(document, "creationInfo") + if len(creation_nodes) > 1: + raise SBOMLineageError("XML singleton element is duplicated") + creation = creation_nodes[0] if creation_nodes else None created_at = _sl_xml_value(creation, "created") creator = _sl_xml_value(creation, "creator") packages = _sl_descendants(document, "Package", "package") @@ -45340,7 +45423,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, _sl_xml_value(raw, "spdxElementId"), _sl_xml_value(raw, "relatedSpdxElement"), _sl_xml_value(raw, "relationshipType", default="related"), - truncated=truncated, component_id_map=component_id_map, + truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id}, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -45352,7 +45435,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) document_id = _sl_spdx_document_id(_sl_xml_text(root, "SPDXID")) - creation = _sl_child(root, "creationInfo") + creation = _sl_single_child(root, "creationInfo") created_at = _sl_xml_text(creation, "created") if creation is not None else "" creator = _sl_xml_text(creation, "creator") if creation is not None else "" packages = _sl_children(root, "package") @@ -45393,7 +45476,7 @@ def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any] relationships.append(_sl_relationship( _sl_xml_text(raw, "spdxElementId"), _sl_xml_text(raw, "relatedSpdxElement"), _sl_xml_text(raw, "relationshipType", default="related"), - truncated=truncated, component_id_map=component_id_map, + truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id}, )) return _sl_finalize_document( fmt="SPDX", spec_version=version, document_id=document_id, @@ -45409,7 +45492,7 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo purl = _sl_xml_text(raw, "purl") if purl: identifiers.append(_sl_safe_locator(purl, "purl")) - external = _sl_child(raw, "externalReferences") + external = _sl_single_child(raw, "externalReferences") if external is not None: reference_nodes = _sl_children(external, "reference") if len(reference_nodes) > _SL_MAX_REFERENCES: @@ -45423,7 +45506,7 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo truncated=component_truncated, )) licenses = [] - license_container = _sl_child(raw, "licenses") + license_container = _sl_single_child(raw, "licenses") if license_container is not None: license_nodes = _sl_children(license_container, "license") if len(license_nodes) > _SL_MAX_LICENSES: @@ -45434,12 +45517,12 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo truncated.extend(item for item in component_truncated if item not in truncated) component = _sl_component( component_id=raw.attrib.get("bom-ref"), name=_sl_xml_text(raw, "name"), - version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_child(raw, "supplier"), "name"), + version=_sl_xml_text(raw, "version"), supplier=_sl_xml_text(_sl_single_child(raw, "supplier"), "name"), identifiers=identifiers, references=refs, component_type=raw.attrib.get("type"), licenses=licenses, truncated=component_truncated, component_id_map=component_id_map, ) if component_id_map is not None and purl: - component_id_map.setdefault(purl, component["component_id"]) + _sl_bind_component_alias_variants(component_id_map, purl, component["component_id"]) return component @@ -45449,13 +45532,13 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if not match: raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") version = _sl_validate_cdx_version(match.group(1)) - metadata = _sl_child(root, "metadata") - metadata_component = _sl_child(metadata, "component") if metadata is not None else None + metadata = _sl_single_child(root, "metadata") + metadata_component = _sl_single_child(metadata, "component") if metadata is not None else None raw_components = [] truncated: list[str] = [] if metadata_component is not None: raw_components.append(metadata_component) - components_node = _sl_child(root, "components") + components_node = _sl_single_child(root, "components") if components_node is not None: component_nodes = _sl_children(components_node, "component") component_budget = _SL_MAX_COMPONENTS - (1 if metadata_component is not None else 0) @@ -45466,8 +45549,12 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, components = [] for raw in raw_components: components.append(_sl_cdx_xml_component(raw, truncated=truncated, component_id_map=component_id_map)) + if "serialNumber" in root.attrib: + document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber") + else: + document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() relationships = [] - dependencies_node = _sl_child(root, "dependencies") + dependencies_node = _sl_single_child(root, "dependencies") if dependencies_node is not None: dependency_nodes = _sl_children(dependencies_node, "dependency") if len(dependency_nodes) > _SL_MAX_RELATIONSHIPS: @@ -45485,14 +45572,11 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if len(children) > allowed_targets: truncated.append("dependency_edges") for child in children[:allowed_targets]: - relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated, component_id_map=component_id_map)) + relationships.append(_sl_relationship(source, child.attrib.get("ref"), "depends_on", truncated=truncated, component_id_map=component_id_map, reserved_ids={document_id})) timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" - authors = _sl_child(metadata, "authors") if metadata is not None else None - author = _sl_xml_text(_sl_child(authors, "author"), "name") if authors is not None else "" - if "serialNumber" in root.attrib: - document_id = _sl_id(root.attrib.get("serialNumber"), "serialNumber") - else: - document_id = "document:sha256:" + hashlib.sha256(raw_bytes).hexdigest() + authors = _sl_single_child(metadata, "authors") if metadata is not None else None + author_nodes = _sl_children(authors, "author") if authors is not None else [] + author = _sl_supplier([_sl_xml_text(item, "name") for item in author_nodes]) if author_nodes else "" return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, @@ -45512,7 +45596,7 @@ def _sl_read_bounded(path: Path) -> bytes: except SBOMLineageError: raise except OSError as exc: - raise SBOMLineageError(f"could not open SBOM safely: {exc}") from exc + raise SBOMLineageError("could not open SBOM safely") from exc try: opened = _sl_os.fstat(fd) if not _sl_stat.S_ISREG(opened.st_mode): @@ -45535,7 +45619,7 @@ def _sl_read_bounded(path: Path) -> bytes: except SBOMLineageError: raise except OSError as exc: - raise SBOMLineageError(f"could not read SBOM safely: {exc}") from exc + raise SBOMLineageError("could not read SBOM safely") from exc finally: _sl_os.close(fd) @@ -45575,9 +45659,18 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) -def _sl_load_json_bounded(source: Path | bytes, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: +def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: """Load JSON through the shared byte and nesting bounds.""" - raw_bytes = _sl_read_bounded(source) if isinstance(source, Path) else bytes(source) + if isinstance(source, Path): + raw_bytes = _sl_read_bounded(source) + elif isinstance(source, bytes): + raw_bytes = source + elif isinstance(source, (bytearray, memoryview)): + raw_bytes = bytes(source) + else: + raise SBOMLineageError("bounded JSON input must be bytes or a regular file") + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") _sl_validate_json_depth(raw_bytes) try: value = json.loads( @@ -45634,6 +45727,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = document elif isinstance(document, bytearray): raw_bytes = bytes(document) + elif isinstance(document, memoryview): + raw_bytes = document.tobytes() elif isinstance(document, Mapping): try: raw_bytes = _sl_json(document).encode("utf-8") @@ -45676,9 +45771,13 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An value, raw_bytes = _sl_payload(document) normalized_source = _sl_safe_source_ref(source_ref, raw_bytes) if isinstance(value, Mapping): - if value.get("spdxVersion") is not None: + has_spdx_marker = "spdxVersion" in value + has_cdx_marker = "bomFormat" in value + if has_spdx_marker and has_cdx_marker: + raise SBOMLineageError("conflicting SBOM format markers") + if has_spdx_marker: return _sl_parse_spdx_json(value, raw_bytes, normalized_source) - if value.get("bomFormat") is not None: + if has_cdx_marker: return _sl_parse_cdx_json(value, raw_bytes, normalized_source) raise SBOMLineageError("SBOM format is missing or unsupported") root_name = _sl_local(value).casefold() @@ -45694,7 +45793,12 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An def _sl_node_kind(node_id: str) -> str: - if node_id == "SPDXRef-DOCUMENT" or node_id.startswith("SPDXRef-DOCUMENT-") or node_id.startswith("document:"): + if ( + node_id == "SPDXRef-DOCUMENT" + or node_id.startswith("SPDXRef-DOCUMENT-") + or node_id.startswith("document:") + or node_id.startswith("urn:uuid:") + ): return "document" if node_id.startswith("artifact:"): return "artifact" @@ -45848,8 +45952,7 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") def _sl_expected_document_coverage(document_id: str, fmt: str, document_name: str | None, created_at: str | None, supplier: str | None, components: list[dict[str, Any]], relationships: list[dict[str, Any]], truncated: list[str]) -> dict[str, Any]: component_ids = {item["component_id"] for item in components} known_ids = set(component_ids) - if fmt == "SPDX": - known_ids.add(document_id) + known_ids.add(document_id) dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) unknown: list[str] = [] if not components: @@ -45911,8 +46014,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non document_id_text = _sl_strict_text(document.get("document_id"), "document_id", limit=256) assert document_id_text is not None document_id = _sl_id(document_id_text, "document_id") - if fmt == "SPDX" and _sl_node_kind(document_id) != "document": - raise SBOMLineageError("SPDX document ID must use the document namespace") + if _sl_node_kind(document_id) != "document": + raise SBOMLineageError("document ID must use the document namespace") for field, limit in (("document_name", 256), ("created_at", 128), ("supplier", 512)): _sl_strict_text(document.get(field), field, allow_none=True, limit=limit) document_sha = document.get("document_sha256") @@ -45933,8 +46036,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non raise SBOMLineageError("normalized SBOM contains duplicate component IDs") seen.add(checked["component_id"]) checked_components.append(checked) - if fmt == "SPDX" and document_id in seen: - raise SBOMLineageError("SPDX document ID collides with a component ID") + if document_id in seen: + raise SBOMLineageError("document ID collides with a component ID") checked_relationships = [_sl_validate_relationship(item) for item in relationships] coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") @@ -45968,7 +46071,7 @@ def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = try: checked = _sl_validate_document(document) if raw_document is None else _sl_rebind_document(document, raw_document) except (SBOMLineageError, TypeError, ValueError) as exc: - return {"valid": False, "error": str(exc)} + return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} @@ -46044,6 +46147,31 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any raise SBOMLineageError("raw_documents may only rebind normalized SBOM projections") normalized.append(ingest_sbom_document(document)) nodes: dict[str, dict[str, Any]] = {} + document_ids: set[str] = set() + component_ids_all = { + component["component_id"] + for document in normalized + for component in document.get("components", []) + } + for document in normalized: + document_id = document["document_id"] + if document_id in document_ids: + raise SBOMLineageError("duplicate document ID") + if document_id in component_ids_all: + raise SBOMLineageError("document ID collides with a component ID") + document_ids.add(document_id) + if len(nodes) >= _SL_MAX_NODES: + raise SBOMLineageError("lineage nodes exceed its global bound") + document_coverage = document["coverage"] + nodes[document_id] = { + "node_id": document_id, + "kind": "document", + "coverage": { + "state": document_coverage["state"], + "unknown": [] if document_coverage["state"] == "complete" else ["document_metadata"], + "truncated": document_coverage["truncated"], + }, + } component_fingerprints: dict[str, str] = {} native_edges: list[dict[str, Any]] = [] for document in normalized: @@ -46051,7 +46179,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any component_id = component["component_id"] fingerprint = _sl_sha(component) if component_id in component_fingerprints and component_fingerprints[component_id] != fingerprint: - raise SBOMLineageError(f"conflicting duplicate component ID: {component_id}") + raise SBOMLineageError("conflicting duplicate component ID") if component_id in component_fingerprints: continue if len(nodes) >= _SL_MAX_NODES: @@ -46099,19 +46227,28 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any return body -def _sl_validate_node_coverage(value: Any, *, component: bool) -> dict[str, Any]: +def _sl_validate_node_coverage(value: Any, *, component: bool, document: bool = False, expected: Mapping[str, Any] | None = None) -> dict[str, Any]: _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "node.coverage") state = value.get("state") if state not in _SL_COVERAGE: raise SBOMLineageError("node.coverage.state is invalid") unknown = _sl_string_list(value.get("unknown"), "node.coverage.unknown", maximum=32) truncated = _sl_string_list(value.get("truncated"), "node.coverage.truncated", maximum=32) - if not component and (state != "partial" or unknown != ["node_metadata"] or truncated): + if document: + if expected is None or {"state", "unknown", "truncated"} != set(expected): + raise SBOMLineageError("document node coverage binding is invalid") + if {"state": state, "unknown": unknown, "truncated": truncated} != dict(expected): + raise SBOMLineageError("document node coverage is not bound to its document") + elif not component and (state != "partial" or unknown != ["node_metadata"] or truncated): raise SBOMLineageError("external lineage node coverage is invalid") return {"state": state, "unknown": unknown, "truncated": truncated} -def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[str, Mapping[str, Any]]]]) -> str: +def _sl_validate_lineage_node( + node: Any, + component_map: Mapping[str, list[tuple[str, Mapping[str, Any]]]], + document_map: Mapping[str, Mapping[str, Any]], +) -> str: if not isinstance(node, Mapping): raise SBOMLineageError("lineage node must be an object") node_id = _sl_strict_text(node.get("node_id"), "node.node_id", limit=256) @@ -46121,6 +46258,19 @@ def _sl_validate_lineage_node(node: Any, component_map: Mapping[str, list[tuple[ expected_kind = _sl_node_kind(node_id) if kind != expected_kind: raise SBOMLineageError("lineage node kind does not match its ID") + if kind == "document": + _sl_require_keys(node, {"node_id", "kind", "coverage"}, set(), "document node") + expected_document = document_map.get(node_id) + if expected_document is None: + raise SBOMLineageError("document node is not bound to a source document") + document_coverage = expected_document["coverage"] + expected_node_coverage = { + "state": document_coverage["state"], + "unknown": [] if document_coverage["state"] == "complete" else ["document_metadata"], + "truncated": document_coverage["truncated"], + } + _sl_validate_node_coverage(node.get("coverage"), component=False, document=True, expected=expected_node_coverage) + return node_id component_fields = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} if kind == "component" and component_fields.issubset(set(node)): required = component_fields | {"node_id", "kind", "document_sha256"} @@ -46172,22 +46322,36 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match lineage documents within its bound") checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] + document_map: dict[str, Mapping[str, Any]] = {} component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} + component_ids_all: set[str] = set() for document in checked_documents: + document_id = document["document_id"] + if document_id in document_map: + raise SBOMLineageError("duplicate document ID") + document_map[document_id] = document for component in document["components"]: + component_ids_all.add(component["component_id"]) component_map.setdefault(component["component_id"], []).append((document["document_sha256"], component)) + if set(document_map).intersection(component_ids_all): + raise SBOMLineageError("document ID collides with a component ID") for component_id, candidates in component_map.items(): if len({_sl_sha(item) for _, item in candidates}) > 1: - raise SBOMLineageError(f"conflicting component projection for {component_id}") + raise SBOMLineageError("conflicting component projection") node_ids: set[str] = set() full_component_ids: set[str] = set() + full_document_ids: set[str] = set() for node in nodes: - node_id = _sl_validate_lineage_node(node, component_map) + node_id = _sl_validate_lineage_node(node, component_map, document_map) if node_id in node_ids: raise SBOMLineageError("lineage contains duplicate nodes") node_ids.add(node_id) + if isinstance(node, Mapping) and node.get("kind") == "document": + full_document_ids.add(node_id) if isinstance(node, Mapping) and "component_id" in node: full_component_ids.add(node_id) + if full_document_ids != set(document_map): + raise SBOMLineageError("lineage document nodes are not bound to all documents") if full_component_ids != set(component_map): raise SBOMLineageError("lineage component nodes are not bound to all documents") checked_edges = [] @@ -46225,7 +46389,7 @@ def verify_sbom_lineage(lineage: Mapping[str, Any], raw_documents: Any | None = try: loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: - return {"valid": False, "error": str(exc)} + return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -46403,7 +46567,13 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin authoritative_lineage = json.loads(authority[0]) except (TypeError, json.JSONDecodeError) as exc: raise SBOMLineageError("authoritative lineage receipt is malformed") from exc + assert authoritative_lineage is not None authority_node_ids: set[str] = set() + authority_document_map = { + document.get("document_id"): document + for document in authoritative_lineage.get("documents", []) + if isinstance(document, Mapping) and isinstance(document.get("document_id"), str) + } for node in lineage_nodes: if not isinstance(node, Mapping): raise SBOMLineageError("query authoritative node must be an object") @@ -46415,6 +46585,20 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin if kind != _sl_node_kind(node_id) or node_id in authority_node_ids: raise SBOMLineageError("query authoritative node identity is invalid") authority_node_ids.add(node_id) + if kind == "document": + document = authority_document_map.get(node_id) + if document is None: + raise SBOMLineageError("query document node is not bound to authoritative documents") + document_coverage = document.get("coverage") + if not isinstance(document_coverage, Mapping): + raise SBOMLineageError("query document node coverage is invalid") + expected_node_coverage = { + "state": document_coverage.get("state"), + "unknown": [] if document_coverage.get("state") == "complete" else ["document_metadata"], + "truncated": document_coverage.get("truncated"), + } + _sl_validate_node_coverage(node.get("coverage"), component=False, document=True, expected=expected_node_coverage) + continue full_component = {"component_id", "name", "version", "supplier", "identifiers", "references", "licenses", "component_type", "coverage"} if full_component.issubset(set(node)): _sl_validate_component({key: node[key] for key in full_component}) @@ -46531,7 +46715,7 @@ def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lin try: checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents) except (SBOMLineageError, TypeError, ValueError) as exc: - return {"valid": False, "error": str(exc)} + return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} @@ -46561,6 +46745,7 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: output = getattr(args, "output", None) if command == "ingest": document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) + _sl_validate_document(document) elif command == "merge": document_paths = _sl_cli_bounded_paths(getattr(args, "documents", None), "documents", required=True) raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") @@ -46583,6 +46768,7 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: else: edges = [] document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) + _sl_loaded_lineage(document, raw_documents=raw_inputs) elif command == "query": raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage, _ = _sl_load_json_bounded(Path(args.lineage)) @@ -46597,6 +46783,7 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: document = query_sbom_lineage( lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, ) + _sl_validate_query_result(document, lineage, raw_inputs) else: raise SBOMLineageError("command must be ingest, merge, or query") serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" @@ -46608,11 +46795,26 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: digest_key = "lineage_digest" if "lineage_digest" in document else "query_digest" if "query_digest" in document else "ingestion_digest" print(f"sbom {command} -> {output}\n{digest_key}: {document[digest_key]}") return 0 - except (OSError, TypeError, ValueError, SBOMLineageError) as exc: + except SBOMLineageError as exc: + error = str(exc)[:160] + if getattr(args, "json", False): + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": error, "valid": False}, sort_keys=True)) + else: + print(f"sbom: {error}") + return 1 + except OSError: + error = "SBOM filesystem operation failed" + if getattr(args, "json", False): + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": error, "valid": False}, sort_keys=True)) + else: + print(f"sbom: {error}") + return 1 + except (TypeError, ValueError): + error = "SBOM input is invalid" if getattr(args, "json", False): - print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": str(exc), "valid": False}, sort_keys=True)) + print(json.dumps({"command": getattr(args, "sbom_command", ""), "error": error, "valid": False}, sort_keys=True)) else: - print(f"sbom: {exc}") + print(f"sbom: {error}") return 1 # ───────────────────── Prompt-size forensics (#606) ────────────────────────── # From a06f464ee21a04c1f49bf8f1eeebbe534aa7b866 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 21:50:15 +0000 Subject: [PATCH 11/40] fix(sbom): close authoritative lineage blockers --- perseus.py | 296 ++++++++++++++++++++++++++++++++---- src/perseus/sbom_lineage.py | 294 +++++++++++++++++++++++++++++++---- tests/test_sbom_lineage.py | 237 ++++++++++++++++++++++++++++- 3 files changed, 765 insertions(+), 62 deletions(-) diff --git a/perseus.py b/perseus.py index a17b0176..e8793ac4 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "8d839d0" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "fcd2773-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44452,7 +44452,7 @@ def cmd_code_map(args, cfg) -> int: import stat as _sl_stat import xml.etree.ElementTree as _sl_et from urllib.parse import unquote -from collections import deque +from collections import OrderedDict, deque from pathlib import Path from typing import Any, Mapping @@ -44485,6 +44485,8 @@ def cmd_code_map(args, cfg) -> int: _SL_MAX_RELATIONSHIPS = 1024 _SL_MAX_DOCUMENTS = 64 _SL_MAX_EDGES = 4096 +_SL_MAX_CLI_TOTAL_BYTES = 64 * 1024 * 1024 +_SL_MAX_CLI_PATH_CHARS = 4096 _SL_MAX_REFERENCES = 64 _SL_MAX_PROPERTIES = 64 _SL_MAX_IDENTIFIERS = 64 @@ -44523,8 +44525,53 @@ def cmd_code_map(args, cfg) -> int: # In-process provenance for normalized projections. A projection digest alone # is caller-recomputable; only a projection produced by raw ingestion (or one # explicitly re-verified against raw bytes) is accepted by lineage builders. -_SL_INGESTED_PROVENANCE: dict[str, tuple[str, str]] = {} -_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str, str]] = {} +_SL_MAX_PROVENANCE_ENTRIES = 128 +_SL_MAX_PROVENANCE_BYTES = 16 * 1024 * 1024 + + +class _SLBoundedReceiptCache(OrderedDict[str, Any]): + """Lifecycle-scoped receipt cache with entry and serialized-byte bounds.""" + + def __init__(self, *, max_entries: int, max_bytes: int) -> None: + super().__init__() + self._max_entries = max_entries + self._max_bytes = max_bytes + self._entry_bytes: dict[str, int] = {} + self._total_bytes = 0 + + @staticmethod + def _size(value: Any) -> int: + return len(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False).encode("utf-8")) + + def __setitem__(self, key: str, value: Any) -> None: + if key in self: + self.__delitem__(key) + entry_size = self._size(value) + if entry_size > self._max_bytes: + return + super().__setitem__(key, value) + self._entry_bytes[key] = entry_size + self._total_bytes += entry_size + while len(self) > self._max_entries or self._total_bytes > self._max_bytes: + old_key, _ = super().popitem(last=False) + self._total_bytes -= self._entry_bytes.pop(old_key, 0) + + def __delitem__(self, key: str) -> None: + super().__delitem__(key) + self._total_bytes -= self._entry_bytes.pop(key, 0) + + def clear(self) -> None: + super().clear() + self._entry_bytes.clear() + self._total_bytes = 0 + + +_SL_INGESTED_PROVENANCE: _SLBoundedReceiptCache = _SLBoundedReceiptCache( + max_entries=_SL_MAX_PROVENANCE_ENTRIES, max_bytes=_SL_MAX_PROVENANCE_BYTES, +) +_SL_LINEAGE_PROVENANCE: _SLBoundedReceiptCache = _SLBoundedReceiptCache( + max_entries=_SL_MAX_PROVENANCE_ENTRIES, max_bytes=_SL_MAX_PROVENANCE_BYTES, +) class SBOMLineageError(ValueError): @@ -44555,6 +44602,25 @@ def _sl_reject_json_constant(value: str) -> None: raise SBOMLineageError("non-finite JSON constant is not allowed") +def _sl_parse_json_int(value: str) -> int: + if len(value.lstrip("-")) > 4096: + raise SBOMLineageError("SBOM JSON integer exceeds its bound") + try: + return int(value) + except (OverflowError, ValueError): + raise SBOMLineageError("SBOM JSON integer is invalid") from None + + +def _sl_parse_json_float(value: str) -> float: + try: + result = float(value) + except (OverflowError, ValueError): + raise SBOMLineageError("SBOM JSON number is invalid") from None + if not math.isfinite(result): + raise SBOMLineageError("non-finite JSON number is not allowed") + return result + + def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() @@ -44817,8 +44883,8 @@ def _sl_xml_scalar(element: Any, names: tuple[str, ...], *, default: str = "") - scalar = str(value).lstrip("#") break values.append(scalar) - if values and any(value != values[0] for value in values[1:]): - raise SBOMLineageError("XML singleton values conflict") + if len(values) > 1: + raise SBOMLineageError("XML singleton element is duplicated") return values[0] if values else default @@ -44839,6 +44905,25 @@ def _sl_validate_xml_tree(root: Any) -> None: stack.extend((child, depth + 1) for child in list(element)) +def _sl_validate_xml_discriminators(root: Any) -> None: + spdx_markers = { + "spdxdocument", "spdxversion", "spdxid", "package", "relationship", "creationinfo", + } + cdx_markers = { + "bom", "bomformat", "specversion", "metadata", "components", "component", "dependencies", "dependency", + } + families: set[str] = set() + for element in root.iter(): + local = _sl_local(element).casefold() + namespace = str(getattr(element, "tag", "")).split("}", 1)[0].casefold() + if local in spdx_markers or "spdx.org" in namespace: + families.add("SPDX") + if local in cdx_markers or "cyclonedx.org" in namespace: + families.add("CycloneDX") + if len(families) > 1: + raise SBOMLineageError("conflicting XML SBOM format markers") + + def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) @@ -44988,6 +45073,35 @@ def _sl_bind_component_alias_variants(component_id_map: dict[str, str], alias: s _sl_bind_component_alias(component_id_map, candidate, normalized_id) +def _sl_component_alias_map(documents: list[Mapping[str, Any]]) -> dict[str, str]: + aliases: dict[str, str] = {} + for document in documents: + for component in document.get("components", []): + component_id = component["component_id"] + _sl_bind_component_alias_variants(aliases, component_id, component_id) + for identifier in component.get("identifiers", []): + _sl_bind_component_alias_variants(aliases, identifier, component_id) + return aliases + + +def _sl_resolve_external_edge_aliases(edges: list[Any], aliases: Mapping[str, str]) -> list[Any]: + resolved: list[Any] = [] + for raw in edges: + if not isinstance(raw, Mapping): + resolved.append(raw) + continue + item: dict[str, Any] = {} + for key in ("from", "source", "to", "target", "type", "confidence", "coverage", "evidence_refs"): + if key in raw: + item[key] = raw[key] + for primary, fallback in (("from", "source"), ("to", "target")): + key = primary if primary in item else fallback if fallback in item else None + if key is not None and isinstance(item.get(key), str): + item[key] = aliases.get(item[key], item[key]) + resolved.append(item) + return resolved + + def _sl_component( *, component_id: Any, @@ -45091,6 +45205,12 @@ def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confid target = component_id_map[target] source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") + if source_id == target_id and _sl_node_kind(source_id) == "document": + raise SBOMLineageError("document self-edge is not allowed") + if not _sl_edge_direction_allowed(source_id, target_id): + if _sl_node_kind(source_id) == "document" or _sl_node_kind(target_id) == "document": + raise SBOMLineageError("document relationship endpoint is not bound") + raise SBOMLineageError("lineage edge direction is not allowed") rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") if not isinstance(confidence, str) or confidence not in _SL_CONFIDENCE: raise SBOMLineageError("relationship confidence is unsupported") @@ -45181,6 +45301,17 @@ def _sl_spdx_document_id(value: Any) -> str: return document_id +def _sl_reject_unbound_document_edges(relationships: list[Any], document_id: str) -> None: + for relationship in relationships: + source = relationship["from"] + target = relationship["to"] + if source == document_id and target == document_id: + raise SBOMLineageError("document self-edge is not allowed") + for endpoint in (source, target): + if _sl_node_kind(endpoint) == "document" and endpoint != document_id: + raise SBOMLineageError("document relationship endpoint is not bound") + + def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") @@ -45199,6 +45330,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu component_ids.add(component_id) if document_id in component_ids: raise SBOMLineageError("document ID collides with a component ID") + _sl_reject_unbound_document_edges(relationships, document_id) known_ids = set(component_ids) known_ids.add(document_id) dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) @@ -45659,14 +45791,25 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) +def _sl_bounded_byteslike(source: bytes | bytearray | memoryview) -> bytes: + size = source.nbytes if isinstance(source, memoryview) else len(source) + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + try: + raw_bytes = source if isinstance(source, bytes) else bytes(source) + except (BufferError, OverflowError, TypeError, ValueError): + raise SBOMLineageError("SBOM bytes-like input is invalid") from None + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return raw_bytes + + def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: """Load JSON through the shared byte and nesting bounds.""" if isinstance(source, Path): raw_bytes = _sl_read_bounded(source) - elif isinstance(source, bytes): - raw_bytes = source - elif isinstance(source, (bytearray, memoryview)): - raw_bytes = bytes(source) + elif isinstance(source, (bytes, bytearray, memoryview)): + raw_bytes = _sl_bounded_byteslike(source) else: raise SBOMLineageError("bounded JSON input must be bytes or a regular file") if len(raw_bytes) > _SL_MAX_INPUT_BYTES: @@ -45676,6 +45819,8 @@ def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allo value = json.loads( raw_bytes.decode("utf-8"), object_pairs_hook=_sl_json_object_pairs, + parse_int=_sl_parse_json_int, + parse_float=_sl_parse_json_float, parse_constant=_sl_reject_json_constant, ) _sl_validate_json_values(value) @@ -45723,12 +45868,8 @@ def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: def _sl_payload(document: Any) -> tuple[Any, bytes]: if isinstance(document, Path): return _sl_payload(_sl_read_bounded(document)) - if isinstance(document, bytes): - raw_bytes = document - elif isinstance(document, bytearray): - raw_bytes = bytes(document) - elif isinstance(document, memoryview): - raw_bytes = document.tobytes() + if isinstance(document, (bytes, bytearray, memoryview)): + raw_bytes = _sl_bounded_byteslike(document) elif isinstance(document, Mapping): try: raw_bytes = _sl_json(document).encode("utf-8") @@ -45750,6 +45891,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: value = json.loads( raw_bytes.decode("utf-8"), object_pairs_hook=_sl_json_object_pairs, + parse_int=_sl_parse_json_int, + parse_float=_sl_parse_json_float, parse_constant=_sl_reject_json_constant, ) _sl_validate_json_values(value) @@ -45780,6 +45923,7 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An if has_cdx_marker: return _sl_parse_cdx_json(value, raw_bytes, normalized_source) raise SBOMLineageError("SBOM format is missing or unsupported") + _sl_validate_xml_discriminators(value) root_name = _sl_local(value).casefold() if root_name == "spdxdocument": if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): @@ -45811,6 +45955,22 @@ def _sl_node_kind(node_id: str) -> str: return "component" +def _sl_edge_direction_allowed(source: str, target: str) -> bool: + source_kind = _sl_node_kind(source) + target_kind = _sl_node_kind(target) + if target_kind == "document": + return False + allowed_targets = { + "document": {"document", "source", "component", "build", "artifact"}, + "source": {"source", "component", "build", "artifact"}, + "component": {"component", "build", "artifact"}, + "build": {"build", "artifact"}, + "artifact": {"deployment"}, + "deployment": set(), + } + return target_kind in allowed_targets[source_kind] + + def _sl_require_keys(value: Any, required: set[str], optional: set[str], field: str) -> None: if not isinstance(value, Mapping) or not required.issubset(set(value)) or set(value) - required - optional: raise SBOMLineageError(f"{field} schema is invalid") @@ -45929,6 +46089,12 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") assert source is not None and target is not None and rel_type is not None _sl_id(source, f"{field}.from") _sl_id(target, f"{field}.to") + if source == target and _sl_node_kind(source) == "document": + raise SBOMLineageError("document self-edge is not allowed") + if not _sl_edge_direction_allowed(source, target): + if _sl_node_kind(source) == "document" or _sl_node_kind(target) == "document": + raise SBOMLineageError(f"{field} document endpoint is not bound") + raise SBOMLineageError(f"{field} direction is not allowed") if rel_type != rel_type.casefold().replace(" ", "_"): raise SBOMLineageError(f"{field}.type is not normalized") if confidence not in _SL_CONFIDENCE or coverage not in _SL_COVERAGE: @@ -46039,6 +46205,7 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non if document_id in seen: raise SBOMLineageError("document ID collides with a component ID") checked_relationships = [_sl_validate_relationship(item) for item in relationships] + _sl_reject_unbound_document_edges(checked_relationships, document_id) coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") if coverage.get("state") not in _SL_COVERAGE: @@ -46075,6 +46242,38 @@ def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} +def _sl_bounded_external_edges(edges: Any) -> list[Any]: + if edges is None: + return [] + if isinstance(edges, (list, tuple)): + try: + if len(edges) > _SL_MAX_EDGES: + raise SBOMLineageError("lineage edges exceed their bound") + except SBOMLineageError: + raise + except Exception: + raise SBOMLineageError("lineage edge collection is invalid") from None + if isinstance(edges, (str, bytes, bytearray, memoryview, Mapping)): + raise SBOMLineageError("lineage edges must be a bounded collection") + try: + iterator = iter(edges) + except Exception: + raise SBOMLineageError("lineage edge collection is invalid") from None + result: list[Any] = [] + try: + for index, raw in enumerate(iterator): + if index >= _SL_MAX_EDGES: + raise SBOMLineageError("lineage edges exceed their bound") + if not isinstance(raw, Mapping): + raise SBOMLineageError("lineage edges must contain objects") + result.append(raw) + except SBOMLineageError: + raise + except Exception: + raise SBOMLineageError("lineage edge collection is invalid") from None + return result + + def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: if edges is None: return [] @@ -46193,10 +46392,16 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any native_edges.extend(document.get("relationships", [])) for marker in document.get("coverage", {}).get("truncated", []): _sl_truncated(truncated, marker) - external_edges = [] if edges is None else list(edges) if isinstance(edges, (list, tuple)) else None - if external_edges is None: - raise SBOMLineageError("lineage edges must be a list") + external_edges = _sl_bounded_external_edges(edges) + component_aliases = _sl_component_alias_map(normalized) + external_edges = _sl_resolve_external_edge_aliases(external_edges, component_aliases) all_edges = _sl_lineage_edges(native_edges + external_edges, truncated=truncated) + for edge in all_edges: + if edge["from"] == edge["to"] and _sl_node_kind(edge["from"]) == "document": + raise SBOMLineageError("document self-edge is not allowed") + for node_id in (edge["from"], edge["to"]): + if _sl_node_kind(node_id) == "document" and node_id not in document_ids: + raise SBOMLineageError("document relationship endpoint is not bound") for edge in all_edges: for node_id in (edge["from"], edge["to"]): if node_id not in nodes and len(nodes) >= _SL_MAX_NODES: @@ -46449,9 +46654,6 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 continue source, target = str(edge.get("from")), str(edge.get("to")) adjacency.setdefault(source, []).append((target, edge)) - reverse = dict(edge) - reverse["from"], reverse["to"] = target, source - adjacency.setdefault(target, []).append((source, reverse)) found: dict[str, dict[str, Any]] = {} path_truncated = False states_used = 0 @@ -46644,9 +46846,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("impacted_artifact.path is outside its bounds") checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] for path_edge in checked_path: - reversed_edge = dict(path_edge) - reversed_edge["from"], reversed_edge["to"] = path_edge["to"], path_edge["from"] - if not any(path_edge == authority_edge or reversed_edge == authority_edge for authority_edge in authority_edges): + if not any(path_edge == authority_edge for authority_edge in authority_edges): raise SBOMLineageError("query path edge is not bound to the authoritative lineage") if checked_path[0]["from"] not in set(matched_nodes): raise SBOMLineageError("query path is not bound to a matched node") @@ -46734,23 +46934,59 @@ def _sl_cli_bounded_paths(value: Any, field: str, *, required: bool = False) -> for item in value: if not isinstance(item, str) or not item: raise SBOMLineageError(f"{field} paths must be strings") + if len(item) > _SL_MAX_CLI_PATH_CHARS: + raise SBOMLineageError(f"{field} path exceeds its character bound") result.append(item) return result +def _sl_cli_checked_path(value: Any, field: str) -> str: + if not isinstance(value, str) or not value: + raise SBOMLineageError(f"{field} path must be a string") + if len(value) > _SL_MAX_CLI_PATH_CHARS: + raise SBOMLineageError(f"{field} path exceeds its character bound") + return value + + +def _sl_cli_preflight_paths(groups: tuple[tuple[str, list[str] | None], ...]) -> None: + total_bytes = 0 + for field, paths in groups: + for path in paths or []: + try: + info = _sl_os.lstat(path) + except OSError as exc: + raise SBOMLineageError("SBOM filesystem operation failed") from exc + if not _sl_stat.S_ISREG(info.st_mode): + raise SBOMLineageError("SBOM CLI input must be a regular file") + size = int(info.st_size) + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError("SBOM CLI input exceeds the per-file byte bound") + total_bytes += size + if total_bytes > _SL_MAX_CLI_TOTAL_BYTES: + raise SBOMLineageError("SBOM CLI input exceeds the aggregate byte bound") + + def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: """CLI adapter for local SBOM normalization, merge, and query operations.""" try: command = getattr(args, "sbom_command", None) output = getattr(args, "output", None) if command == "ingest": - document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) + input_path = _sl_cli_checked_path(getattr(args, "document", None), "document") + _sl_cli_preflight_paths((("document", [input_path]),)) + document = ingest_sbom_document(Path(input_path), source_ref=getattr(args, "source_ref", "")) _sl_validate_document(document) elif command == "merge": document_paths = _sl_cli_bounded_paths(getattr(args, "documents", None), "documents", required=True) raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") + edge_path = _sl_cli_checked_path(getattr(args, "edges", None), "edges") if getattr(args, "edges", None) else None if raw_document_paths is not None and len(raw_document_paths) != len(document_paths or []): raise SBOMLineageError("raw_documents must contain one raw source per document") + _sl_cli_preflight_paths(( + ("documents", document_paths), + ("raw_documents", raw_document_paths), + ("edges", [edge_path] if edge_path is not None else None), + )) documents = [] for path in document_paths or []: payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) @@ -46761,8 +46997,8 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: for raw_path in raw_document_paths: _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) raw_inputs.append(raw_bytes) - if getattr(args, "edges", None): - edges, _ = _sl_load_json_bounded(Path(args.edges)) + if edge_path is not None: + edges, _ = _sl_load_json_bounded(Path(edge_path)) if not isinstance(edges, list): raise SBOMLineageError("lineage edges JSON must be a list") else: @@ -46771,7 +47007,9 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: _sl_loaded_lineage(document, raw_documents=raw_inputs) elif command == "query": raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") - lineage, _ = _sl_load_json_bounded(Path(args.lineage)) + lineage_path = _sl_cli_checked_path(getattr(args, "lineage", None), "lineage") + _sl_cli_preflight_paths((("lineage", [lineage_path]), ("raw_documents", raw_document_paths))) + lineage, _ = _sl_load_json_bounded(Path(lineage_path)) if not isinstance(lineage, Mapping): raise SBOMLineageError("lineage JSON root must be an object") raw_inputs = None diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 48a667b5..eeeb1c90 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -16,7 +16,7 @@ import stat as _sl_stat import xml.etree.ElementTree as _sl_et from urllib.parse import unquote -from collections import deque +from collections import OrderedDict, deque from pathlib import Path from typing import Any, Mapping @@ -49,6 +49,8 @@ _SL_MAX_RELATIONSHIPS = 1024 _SL_MAX_DOCUMENTS = 64 _SL_MAX_EDGES = 4096 +_SL_MAX_CLI_TOTAL_BYTES = 64 * 1024 * 1024 +_SL_MAX_CLI_PATH_CHARS = 4096 _SL_MAX_REFERENCES = 64 _SL_MAX_PROPERTIES = 64 _SL_MAX_IDENTIFIERS = 64 @@ -87,8 +89,53 @@ # In-process provenance for normalized projections. A projection digest alone # is caller-recomputable; only a projection produced by raw ingestion (or one # explicitly re-verified against raw bytes) is accepted by lineage builders. -_SL_INGESTED_PROVENANCE: dict[str, tuple[str, str]] = {} -_SL_LINEAGE_PROVENANCE: dict[str, tuple[str, str, str]] = {} +_SL_MAX_PROVENANCE_ENTRIES = 128 +_SL_MAX_PROVENANCE_BYTES = 16 * 1024 * 1024 + + +class _SLBoundedReceiptCache(OrderedDict[str, Any]): + """Lifecycle-scoped receipt cache with entry and serialized-byte bounds.""" + + def __init__(self, *, max_entries: int, max_bytes: int) -> None: + super().__init__() + self._max_entries = max_entries + self._max_bytes = max_bytes + self._entry_bytes: dict[str, int] = {} + self._total_bytes = 0 + + @staticmethod + def _size(value: Any) -> int: + return len(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True, allow_nan=False).encode("utf-8")) + + def __setitem__(self, key: str, value: Any) -> None: + if key in self: + self.__delitem__(key) + entry_size = self._size(value) + if entry_size > self._max_bytes: + return + super().__setitem__(key, value) + self._entry_bytes[key] = entry_size + self._total_bytes += entry_size + while len(self) > self._max_entries or self._total_bytes > self._max_bytes: + old_key, _ = super().popitem(last=False) + self._total_bytes -= self._entry_bytes.pop(old_key, 0) + + def __delitem__(self, key: str) -> None: + super().__delitem__(key) + self._total_bytes -= self._entry_bytes.pop(key, 0) + + def clear(self) -> None: + super().clear() + self._entry_bytes.clear() + self._total_bytes = 0 + + +_SL_INGESTED_PROVENANCE: _SLBoundedReceiptCache = _SLBoundedReceiptCache( + max_entries=_SL_MAX_PROVENANCE_ENTRIES, max_bytes=_SL_MAX_PROVENANCE_BYTES, +) +_SL_LINEAGE_PROVENANCE: _SLBoundedReceiptCache = _SLBoundedReceiptCache( + max_entries=_SL_MAX_PROVENANCE_ENTRIES, max_bytes=_SL_MAX_PROVENANCE_BYTES, +) class SBOMLineageError(ValueError): @@ -119,6 +166,25 @@ def _sl_reject_json_constant(value: str) -> None: raise SBOMLineageError("non-finite JSON constant is not allowed") +def _sl_parse_json_int(value: str) -> int: + if len(value.lstrip("-")) > 4096: + raise SBOMLineageError("SBOM JSON integer exceeds its bound") + try: + return int(value) + except (OverflowError, ValueError): + raise SBOMLineageError("SBOM JSON integer is invalid") from None + + +def _sl_parse_json_float(value: str) -> float: + try: + result = float(value) + except (OverflowError, ValueError): + raise SBOMLineageError("SBOM JSON number is invalid") from None + if not math.isfinite(result): + raise SBOMLineageError("non-finite JSON number is not allowed") + return result + + def _sl_sha(value: Any) -> str: return hashlib.sha256(_sl_json(value).encode("utf-8")).hexdigest() @@ -381,8 +447,8 @@ def _sl_xml_scalar(element: Any, names: tuple[str, ...], *, default: str = "") - scalar = str(value).lstrip("#") break values.append(scalar) - if values and any(value != values[0] for value in values[1:]): - raise SBOMLineageError("XML singleton values conflict") + if len(values) > 1: + raise SBOMLineageError("XML singleton element is duplicated") return values[0] if values else default @@ -403,6 +469,25 @@ def _sl_validate_xml_tree(root: Any) -> None: stack.extend((child, depth + 1) for child in list(element)) +def _sl_validate_xml_discriminators(root: Any) -> None: + spdx_markers = { + "spdxdocument", "spdxversion", "spdxid", "package", "relationship", "creationinfo", + } + cdx_markers = { + "bom", "bomformat", "specversion", "metadata", "components", "component", "dependencies", "dependency", + } + families: set[str] = set() + for element in root.iter(): + local = _sl_local(element).casefold() + namespace = str(getattr(element, "tag", "")).split("}", 1)[0].casefold() + if local in spdx_markers or "spdx.org" in namespace: + families.add("SPDX") + if local in cdx_markers or "cyclonedx.org" in namespace: + families.add("CycloneDX") + if len(families) > 1: + raise SBOMLineageError("conflicting XML SBOM format markers") + + def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) @@ -552,6 +637,35 @@ def _sl_bind_component_alias_variants(component_id_map: dict[str, str], alias: s _sl_bind_component_alias(component_id_map, candidate, normalized_id) +def _sl_component_alias_map(documents: list[Mapping[str, Any]]) -> dict[str, str]: + aliases: dict[str, str] = {} + for document in documents: + for component in document.get("components", []): + component_id = component["component_id"] + _sl_bind_component_alias_variants(aliases, component_id, component_id) + for identifier in component.get("identifiers", []): + _sl_bind_component_alias_variants(aliases, identifier, component_id) + return aliases + + +def _sl_resolve_external_edge_aliases(edges: list[Any], aliases: Mapping[str, str]) -> list[Any]: + resolved: list[Any] = [] + for raw in edges: + if not isinstance(raw, Mapping): + resolved.append(raw) + continue + item: dict[str, Any] = {} + for key in ("from", "source", "to", "target", "type", "confidence", "coverage", "evidence_refs"): + if key in raw: + item[key] = raw[key] + for primary, fallback in (("from", "source"), ("to", "target")): + key = primary if primary in item else fallback if fallback in item else None + if key is not None and isinstance(item.get(key), str): + item[key] = aliases.get(item[key], item[key]) + resolved.append(item) + return resolved + + def _sl_component( *, component_id: Any, @@ -655,6 +769,12 @@ def _sl_relationship(source: Any, target: Any, relationship_type: Any, *, confid target = component_id_map[target] source_id = _sl_id(source, "relationship.from") target_id = _sl_id(target, "relationship.to") + if source_id == target_id and _sl_node_kind(source_id) == "document": + raise SBOMLineageError("document self-edge is not allowed") + if not _sl_edge_direction_allowed(source_id, target_id): + if _sl_node_kind(source_id) == "document" or _sl_node_kind(target_id) == "document": + raise SBOMLineageError("document relationship endpoint is not bound") + raise SBOMLineageError("lineage edge direction is not allowed") rel_type = _sl_safe_text(relationship_type, "relationship.type", required=True, limit=96).casefold().replace(" ", "_") if not isinstance(confidence, str) or confidence not in _SL_CONFIDENCE: raise SBOMLineageError("relationship confidence is unsupported") @@ -745,6 +865,17 @@ def _sl_spdx_document_id(value: Any) -> str: return document_id +def _sl_reject_unbound_document_edges(relationships: list[Any], document_id: str) -> None: + for relationship in relationships: + source = relationship["from"] + target = relationship["to"] + if source == document_id and target == document_id: + raise SBOMLineageError("document self-edge is not allowed") + for endpoint in (source, target): + if _sl_node_kind(endpoint) == "document" and endpoint != document_id: + raise SBOMLineageError("document relationship endpoint is not bound") + + def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: if fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") @@ -763,6 +894,7 @@ def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, docu component_ids.add(component_id) if document_id in component_ids: raise SBOMLineageError("document ID collides with a component ID") + _sl_reject_unbound_document_edges(relationships, document_id) known_ids = set(component_ids) known_ids.add(document_id) dangling = sorted({f"{item['from']}->{item['to']}" for item in relationships if item["from"] not in known_ids or item["to"] not in known_ids}) @@ -1223,14 +1355,25 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) +def _sl_bounded_byteslike(source: bytes | bytearray | memoryview) -> bytes: + size = source.nbytes if isinstance(source, memoryview) else len(source) + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + try: + raw_bytes = source if isinstance(source, bytes) else bytes(source) + except (BufferError, OverflowError, TypeError, ValueError): + raise SBOMLineageError("SBOM bytes-like input is invalid") from None + if len(raw_bytes) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return raw_bytes + + def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allow_non_json: bool = False) -> tuple[Any | None, bytes]: """Load JSON through the shared byte and nesting bounds.""" if isinstance(source, Path): raw_bytes = _sl_read_bounded(source) - elif isinstance(source, bytes): - raw_bytes = source - elif isinstance(source, (bytearray, memoryview)): - raw_bytes = bytes(source) + elif isinstance(source, (bytes, bytearray, memoryview)): + raw_bytes = _sl_bounded_byteslike(source) else: raise SBOMLineageError("bounded JSON input must be bytes or a regular file") if len(raw_bytes) > _SL_MAX_INPUT_BYTES: @@ -1240,6 +1383,8 @@ def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allo value = json.loads( raw_bytes.decode("utf-8"), object_pairs_hook=_sl_json_object_pairs, + parse_int=_sl_parse_json_int, + parse_float=_sl_parse_json_float, parse_constant=_sl_reject_json_constant, ) _sl_validate_json_values(value) @@ -1287,12 +1432,8 @@ def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: def _sl_payload(document: Any) -> tuple[Any, bytes]: if isinstance(document, Path): return _sl_payload(_sl_read_bounded(document)) - if isinstance(document, bytes): - raw_bytes = document - elif isinstance(document, bytearray): - raw_bytes = bytes(document) - elif isinstance(document, memoryview): - raw_bytes = document.tobytes() + if isinstance(document, (bytes, bytearray, memoryview)): + raw_bytes = _sl_bounded_byteslike(document) elif isinstance(document, Mapping): try: raw_bytes = _sl_json(document).encode("utf-8") @@ -1314,6 +1455,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: value = json.loads( raw_bytes.decode("utf-8"), object_pairs_hook=_sl_json_object_pairs, + parse_int=_sl_parse_json_int, + parse_float=_sl_parse_json_float, parse_constant=_sl_reject_json_constant, ) _sl_validate_json_values(value) @@ -1344,6 +1487,7 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An if has_cdx_marker: return _sl_parse_cdx_json(value, raw_bytes, normalized_source) raise SBOMLineageError("SBOM format is missing or unsupported") + _sl_validate_xml_discriminators(value) root_name = _sl_local(value).casefold() if root_name == "spdxdocument": if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): @@ -1375,6 +1519,22 @@ def _sl_node_kind(node_id: str) -> str: return "component" +def _sl_edge_direction_allowed(source: str, target: str) -> bool: + source_kind = _sl_node_kind(source) + target_kind = _sl_node_kind(target) + if target_kind == "document": + return False + allowed_targets = { + "document": {"document", "source", "component", "build", "artifact"}, + "source": {"source", "component", "build", "artifact"}, + "component": {"component", "build", "artifact"}, + "build": {"build", "artifact"}, + "artifact": {"deployment"}, + "deployment": set(), + } + return target_kind in allowed_targets[source_kind] + + def _sl_require_keys(value: Any, required: set[str], optional: set[str], field: str) -> None: if not isinstance(value, Mapping) or not required.issubset(set(value)) or set(value) - required - optional: raise SBOMLineageError(f"{field} schema is invalid") @@ -1493,6 +1653,12 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") assert source is not None and target is not None and rel_type is not None _sl_id(source, f"{field}.from") _sl_id(target, f"{field}.to") + if source == target and _sl_node_kind(source) == "document": + raise SBOMLineageError("document self-edge is not allowed") + if not _sl_edge_direction_allowed(source, target): + if _sl_node_kind(source) == "document" or _sl_node_kind(target) == "document": + raise SBOMLineageError(f"{field} document endpoint is not bound") + raise SBOMLineageError(f"{field} direction is not allowed") if rel_type != rel_type.casefold().replace(" ", "_"): raise SBOMLineageError(f"{field}.type is not normalized") if confidence not in _SL_CONFIDENCE or coverage not in _SL_COVERAGE: @@ -1603,6 +1769,7 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non if document_id in seen: raise SBOMLineageError("document ID collides with a component ID") checked_relationships = [_sl_validate_relationship(item) for item in relationships] + _sl_reject_unbound_document_edges(checked_relationships, document_id) coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") if coverage.get("state") not in _SL_COVERAGE: @@ -1639,6 +1806,38 @@ def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} +def _sl_bounded_external_edges(edges: Any) -> list[Any]: + if edges is None: + return [] + if isinstance(edges, (list, tuple)): + try: + if len(edges) > _SL_MAX_EDGES: + raise SBOMLineageError("lineage edges exceed their bound") + except SBOMLineageError: + raise + except Exception: + raise SBOMLineageError("lineage edge collection is invalid") from None + if isinstance(edges, (str, bytes, bytearray, memoryview, Mapping)): + raise SBOMLineageError("lineage edges must be a bounded collection") + try: + iterator = iter(edges) + except Exception: + raise SBOMLineageError("lineage edge collection is invalid") from None + result: list[Any] = [] + try: + for index, raw in enumerate(iterator): + if index >= _SL_MAX_EDGES: + raise SBOMLineageError("lineage edges exceed their bound") + if not isinstance(raw, Mapping): + raise SBOMLineageError("lineage edges must contain objects") + result.append(raw) + except SBOMLineageError: + raise + except Exception: + raise SBOMLineageError("lineage edge collection is invalid") from None + return result + + def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: if edges is None: return [] @@ -1757,10 +1956,16 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any native_edges.extend(document.get("relationships", [])) for marker in document.get("coverage", {}).get("truncated", []): _sl_truncated(truncated, marker) - external_edges = [] if edges is None else list(edges) if isinstance(edges, (list, tuple)) else None - if external_edges is None: - raise SBOMLineageError("lineage edges must be a list") + external_edges = _sl_bounded_external_edges(edges) + component_aliases = _sl_component_alias_map(normalized) + external_edges = _sl_resolve_external_edge_aliases(external_edges, component_aliases) all_edges = _sl_lineage_edges(native_edges + external_edges, truncated=truncated) + for edge in all_edges: + if edge["from"] == edge["to"] and _sl_node_kind(edge["from"]) == "document": + raise SBOMLineageError("document self-edge is not allowed") + for node_id in (edge["from"], edge["to"]): + if _sl_node_kind(node_id) == "document" and node_id not in document_ids: + raise SBOMLineageError("document relationship endpoint is not bound") for edge in all_edges: for node_id in (edge["from"], edge["to"]): if node_id not in nodes and len(nodes) >= _SL_MAX_NODES: @@ -2013,9 +2218,6 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 continue source, target = str(edge.get("from")), str(edge.get("to")) adjacency.setdefault(source, []).append((target, edge)) - reverse = dict(edge) - reverse["from"], reverse["to"] = target, source - adjacency.setdefault(target, []).append((source, reverse)) found: dict[str, dict[str, Any]] = {} path_truncated = False states_used = 0 @@ -2208,9 +2410,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("impacted_artifact.path is outside its bounds") checked_path = [_sl_validate_relationship(edge, field="query.path.edge") for edge in path] for path_edge in checked_path: - reversed_edge = dict(path_edge) - reversed_edge["from"], reversed_edge["to"] = path_edge["to"], path_edge["from"] - if not any(path_edge == authority_edge or reversed_edge == authority_edge for authority_edge in authority_edges): + if not any(path_edge == authority_edge for authority_edge in authority_edges): raise SBOMLineageError("query path edge is not bound to the authoritative lineage") if checked_path[0]["from"] not in set(matched_nodes): raise SBOMLineageError("query path is not bound to a matched node") @@ -2298,23 +2498,59 @@ def _sl_cli_bounded_paths(value: Any, field: str, *, required: bool = False) -> for item in value: if not isinstance(item, str) or not item: raise SBOMLineageError(f"{field} paths must be strings") + if len(item) > _SL_MAX_CLI_PATH_CHARS: + raise SBOMLineageError(f"{field} path exceeds its character bound") result.append(item) return result +def _sl_cli_checked_path(value: Any, field: str) -> str: + if not isinstance(value, str) or not value: + raise SBOMLineageError(f"{field} path must be a string") + if len(value) > _SL_MAX_CLI_PATH_CHARS: + raise SBOMLineageError(f"{field} path exceeds its character bound") + return value + + +def _sl_cli_preflight_paths(groups: tuple[tuple[str, list[str] | None], ...]) -> None: + total_bytes = 0 + for field, paths in groups: + for path in paths or []: + try: + info = _sl_os.lstat(path) + except OSError as exc: + raise SBOMLineageError("SBOM filesystem operation failed") from exc + if not _sl_stat.S_ISREG(info.st_mode): + raise SBOMLineageError("SBOM CLI input must be a regular file") + size = int(info.st_size) + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError("SBOM CLI input exceeds the per-file byte bound") + total_bytes += size + if total_bytes > _SL_MAX_CLI_TOTAL_BYTES: + raise SBOMLineageError("SBOM CLI input exceeds the aggregate byte bound") + + def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: """CLI adapter for local SBOM normalization, merge, and query operations.""" try: command = getattr(args, "sbom_command", None) output = getattr(args, "output", None) if command == "ingest": - document = ingest_sbom_document(Path(args.document), source_ref=getattr(args, "source_ref", "")) + input_path = _sl_cli_checked_path(getattr(args, "document", None), "document") + _sl_cli_preflight_paths((("document", [input_path]),)) + document = ingest_sbom_document(Path(input_path), source_ref=getattr(args, "source_ref", "")) _sl_validate_document(document) elif command == "merge": document_paths = _sl_cli_bounded_paths(getattr(args, "documents", None), "documents", required=True) raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") + edge_path = _sl_cli_checked_path(getattr(args, "edges", None), "edges") if getattr(args, "edges", None) else None if raw_document_paths is not None and len(raw_document_paths) != len(document_paths or []): raise SBOMLineageError("raw_documents must contain one raw source per document") + _sl_cli_preflight_paths(( + ("documents", document_paths), + ("raw_documents", raw_document_paths), + ("edges", [edge_path] if edge_path is not None else None), + )) documents = [] for path in document_paths or []: payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) @@ -2325,8 +2561,8 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: for raw_path in raw_document_paths: _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) raw_inputs.append(raw_bytes) - if getattr(args, "edges", None): - edges, _ = _sl_load_json_bounded(Path(args.edges)) + if edge_path is not None: + edges, _ = _sl_load_json_bounded(Path(edge_path)) if not isinstance(edges, list): raise SBOMLineageError("lineage edges JSON must be a list") else: @@ -2335,7 +2571,9 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: _sl_loaded_lineage(document, raw_documents=raw_inputs) elif command == "query": raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") - lineage, _ = _sl_load_json_bounded(Path(args.lineage)) + lineage_path = _sl_cli_checked_path(getattr(args, "lineage", None), "lineage") + _sl_cli_preflight_paths((("lineage", [lineage_path]), ("raw_documents", raw_document_paths))) + lineage, _ = _sl_load_json_bounded(Path(lineage_path)) if not isinstance(lineage, Mapping): raise SBOMLineageError("lineage JSON root must be an object") raw_inputs = None diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 7c264a88..3a707c5f 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -71,7 +71,7 @@ def test_query_returns_auditable_impacted_artifact_path_without_false_clean_clai assert result["status"] == "partial" assert result["impacted_artifacts"][0]["artifact_id"] == "artifact:perseus-image@1.0.26" assert len(result["impacted_artifacts"][0]["path"]) == 3 - assert [edge["type"] for edge in result["impacted_artifacts"][0]["path"]] == ["depends_on", "built_into", "generates"] + assert [edge["type"] for edge in result["impacted_artifacts"][0]["path"]] == ["used_by", "built_into", "generates"] assert result["impacted_artifacts"][0]["coverage"] == "complete" assert result["coverage"]["state"] == "partial" assert result["claims"]["not_affected"] is False @@ -188,6 +188,7 @@ def test_untrusted_normalized_documents_and_conflicting_duplicate_ids_fail_close perseus.build_sbom_lineage([tampered]) alternate = json.loads(_load("spdx-app.json")) alternate["SPDXID"] = "SPDXRef-DOCUMENT-ALTERNATE" + alternate["relationships"][0]["spdxElementId"] = alternate["SPDXID"] alternate["packages"][0]["name"] = "different-component" alternate_document = perseus.ingest_sbom_document(alternate, source_ref="artifact:alternate") with pytest.raises(perseus.SBOMLineageError, match="duplicate component ID"): @@ -360,9 +361,8 @@ def test_all_collection_caps_are_recorded_and_downgrade_coverage(): assert lineage["coverage"]["state"] != "complete" edges = [{"from": f"source:s{i}", "to": f"artifact:a{i}", "type": "generates", "confidence": "high", "coverage": "complete"} for i in range(4097)] - edge_lineage = perseus.build_sbom_lineage([document], edges=edges) - assert "edges" in edge_lineage["coverage"]["truncated"] - assert edge_lineage["coverage"]["state"] != "complete" + with pytest.raises(perseus.SBOMLineageError, match="edges|bound"): + perseus.build_sbom_lineage([document], edges=edges) def test_query_result_cap_is_recorded_and_cannot_claim_established_impact(): @@ -1207,10 +1207,12 @@ def test_cli_ingest_rejects_a_malformed_normalized_return_before_writing(tmp_pat valid = perseus.ingest_sbom_document(_load("cyclonedx-app.json"), source_ref="artifact:cli-validation") invalid = json.loads(json.dumps(valid)) invalid["components"][0]["supplier"] = "A" * 513 + input_path = tmp_path / "ignored-input.json" + input_path.write_bytes(_load("cyclonedx-app.json")) monkeypatch.setattr(perseus, "ingest_sbom_document", lambda *args, **kwargs: invalid) output = tmp_path / "invalid-normalized.json" args = type("Args", (), { - "sbom_command": "ingest", "document": "ignored-input", "source_ref": "", + "sbom_command": "ingest", "document": str(input_path), "source_ref": "", "output": str(output), "json": True, })() assert perseus.cmd_sbom(args, {}) == 1 @@ -1258,3 +1260,228 @@ def test_known_cyclonedx_document_endpoint_remains_bound_and_complete(): node = next(item for item in lineage["nodes"] if item["node_id"] == document_id) assert node["kind"] == "document" assert perseus.verify_sbom_lineage(lineage)["valid"] is True + + +def test_oversized_ignored_json_integer_is_a_structured_sbom_error(): + raw = _load("cyclonedx-app.json").decode("utf-8").replace( + '"version": 1,', + '"version": 1, "ignored": {"overflow": ' + "9" * 5000 + '},', + ) + with pytest.raises(perseus.SBOMLineageError) as error: + perseus.ingest_sbom_document(raw) + assert "9" * 32 not in str(error.value) + + +def test_external_edges_resolve_component_purl_aliases_across_lineage(): + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["bom-ref"] = "custom:log4j-component" + purl = component["purl"] + document = perseus.ingest_sbom_document(payload, source_ref="artifact:alias-lineage") + component_id = next(item["component_id"] for item in document["components"] if item["name"] == "log4j-core") + lineage = perseus.build_sbom_lineage([document], edges=[ + {"from": purl, "to": "build:alias-build", "type": "built_into", "confidence": "high", "coverage": "complete"}, + {"from": "build:alias-build", "to": "artifact:alias-image", "type": "generates", "confidence": "high", "coverage": "complete"}, + ]) + assert purl not in {node["node_id"] for node in lineage["nodes"]} + result = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + assert result["impacted_artifacts"][0]["artifact_id"] == "artifact:alias-image" + assert result["impacted_artifacts"][0]["path"][0]["from"] == component_id + + +def test_external_aliases_are_rejected_when_normalized_documents_are_ambiguous(): + first_payload = json.loads(_load("cyclonedx-app.json")) + second_payload = json.loads(_load("cyclonedx-app.json")) + shared_purl = "pkg:generic/shared-alias@1" + first_payload["components"][0]["bom-ref"] = "custom:first" + first_payload["components"][0]["purl"] = shared_purl + first_payload["serialNumber"] = "urn:uuid:alias-first" + second_payload["components"][0]["bom-ref"] = "custom:second" + second_payload["components"][0]["purl"] = shared_purl + second_payload["serialNumber"] = "urn:uuid:alias-second" + first = perseus.ingest_sbom_document(first_payload, source_ref="artifact:alias-first") + second = perseus.ingest_sbom_document(second_payload, source_ref="artifact:alias-second") + with pytest.raises(perseus.SBOMLineageError, match="ambiguous|alias"): + perseus.build_sbom_lineage([first, second], edges=[]) + + +def test_reserved_component_collision_cannot_create_a_document_self_edge(): + payload = json.loads(_load("spdx-app.json")) + payload["packages"][0]["SPDXID"] = "SPDXRef-DOCUMENT" + payload["relationships"] = [{ + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-DOCUMENT", + }] + with pytest.raises(perseus.SBOMLineageError, match="self|document|reserved"): + perseus.ingest_sbom_document(payload) + + +def test_unbound_document_namespace_relationship_endpoints_are_rejected(): + payload = json.loads(_load("spdx-app.json")) + payload["relationships"] = [{ + "spdxElementId": "SPDXRef-DOCUMENT", + "relationshipType": "DESCRIBES", + "relatedSpdxElement": "SPDXRef-DOCUMENT-MISSING", + }] + with pytest.raises(perseus.SBOMLineageError, match="document|bound|dangling"): + perseus.ingest_sbom_document(payload) + + +def test_external_edges_cannot_create_unbound_document_nodes(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:document-edge") + with pytest.raises(perseus.SBOMLineageError, match="document|bound"): + perseus.build_sbom_lineage([document], edges=[{ + "from": "document:unbound", + "to": "SPDXRef-App", + "type": "describes", + "confidence": "high", + "coverage": "complete", + }]) + + +def test_xml_singletons_reject_identical_duplicates_and_cross_family_markers(): + spdx_xml = _load("spdx-app.xml").decode("utf-8").replace( + "SPDX-2.3", + "SPDX-2.3SPDX-2.3", + ) + with pytest.raises(perseus.SBOMLineageError, match="singleton|duplicat"): + perseus.ingest_sbom_document(spdx_xml) + + cdx_xml = _load("cyclonedx-app.xml").decode("utf-8").replace( + "", "SPDX-2.3", 1, + ) + with pytest.raises(perseus.SBOMLineageError, match="format|conflict|marker"): + perseus.ingest_sbom_document(cdx_xml) + + spdx_with_cdx_marker = _load("spdx-app.xml").decode("utf-8").replace( + "perseus-build-xml", + "perseus-build-xmlCycloneDX", + ) + with pytest.raises(perseus.SBOMLineageError, match="format|conflict|marker"): + perseus.ingest_sbom_document(spdx_with_cdx_marker) + + +def test_query_does_not_reverse_traverse_artifact_to_component_edges(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:direction-query") + with pytest.raises(perseus.SBOMLineageError, match="direction"): + perseus.build_sbom_lineage([document], edges=[ + {"from": "artifact:upstream", "to": "SPDXRef-Log4j", "type": "feeds_back", "confidence": "high", "coverage": "complete"}, + ]) + + +def test_lineage_verifier_rejects_resealed_reversed_authoritative_edges(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:direction-verify") + lineage = perseus.build_sbom_lineage([document], edges=[ + {"from": "SPDXRef-Log4j", "to": "artifact:direction-target", "type": "generates", "confidence": "high", "coverage": "complete"}, + ]) + forged = json.loads(json.dumps(lineage)) + edge = next(item for item in forged["edges"] if item["to"] == "artifact:direction-target") + edge["from"], edge["to"] = edge["to"], edge["from"] + _reseal(forged) + assert perseus.verify_sbom_lineage(forged, raw_documents=[raw])["valid"] is False + + +def test_cli_aggregate_input_budget_fails_before_retaining_document_buffers(tmp_path, monkeypatch, capsys): + first = tmp_path / "PATH_SECRET_FIRST.json" + second = tmp_path / "PATH_SECRET_SECOND.json" + first.write_bytes(b"{}") + second.write_bytes(b"{}") + calls = [] + + def unexpected_load(*args, **kwargs): + calls.append(args) + raise AssertionError("CLI retained a document before aggregate preflight") + + monkeypatch.setattr(perseus, "_sl_load_json_bounded", unexpected_load) + monkeypatch.setattr(perseus, "_SL_MAX_CLI_TOTAL_BYTES", 3) + args = type("Args", (), { + "sbom_command": "merge", "documents": [str(first), str(second)], + "raw_documents": None, "edges": None, "output": None, "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 1 + assert calls == [] + failure = json.loads(capsys.readouterr().out) + assert failure["valid"] is False + assert "PATH_SECRET" not in json.dumps(failure) + + +def test_cli_path_character_bound_fails_before_stat_or_read(tmp_path, monkeypatch, capsys): + path = tmp_path / "PATH_SECRET_INPUT.json" + path.write_bytes(b"{}") + monkeypatch.setattr(perseus, "_SL_MAX_CLI_PATH_CHARS", len(str(path)) - 1) + args = type("Args", (), { + "sbom_command": "merge", "documents": [str(path)], + "raw_documents": None, "edges": None, "output": None, "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 1 + failure = json.loads(capsys.readouterr().out) + assert failure["valid"] is False + assert "PATH_SECRET" not in json.dumps(failure) + + +def test_bytes_like_bounds_are_checked_before_copy_at_exact_and_over_limits(monkeypatch): + monkeypatch.setattr(perseus, "_SL_MAX_INPUT_BYTES", 2) + exact_values = (b"{}", bytearray(b"{}"), memoryview(b"{}")) + for value in exact_values: + loaded, raw = perseus._sl_load_json_bounded(value) + assert loaded == {} + assert raw == b"{}" + payload, payload_raw = perseus._sl_payload(value) + assert payload == {} + assert payload_raw == b"{}" + + class ExplodingBytearray(bytearray): + def __bytes__(self): + raise AssertionError("oversized bytes-like input was copied") + + for value in (b"{} ", memoryview(b"{} ")): + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_load_json_bounded(value) + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_payload(value) + for loader in (perseus._sl_load_json_bounded, perseus._sl_payload): + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + loader(ExplodingBytearray(b"{} ")) + + +def test_provenance_receipts_are_bounded_for_repeated_ingests_and_lineages(): + max_entries = 128 + raw = _load("spdx-app.json") + perseus._SL_INGESTED_PROVENANCE.clear() + for index in range(max_entries + 7): + perseus.ingest_sbom_document(raw, source_ref=f"artifact:provenance-{index}") + assert len(perseus._SL_INGESTED_PROVENANCE) <= max_entries + + perseus._SL_LINEAGE_PROVENANCE.clear() + document = perseus.ingest_sbom_document(raw, source_ref="artifact:provenance-lineage") + for index in range(max_entries + 7): + perseus.build_sbom_lineage([document], edges=[{ + "from": f"source:provenance-{index}", + "to": f"artifact:provenance-{index}", + "type": "generates", + "confidence": "high", + "coverage": "complete", + }]) + assert len(perseus._SL_LINEAGE_PROVENANCE) <= max_entries + + +def test_external_edge_iterators_are_bounded_and_oversized_collections_rejected(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:edge-contract") + + def one_edge(): + yield {"from": "source:iterator", "to": "artifact:iterator", "type": "generates", "confidence": "high", "coverage": "complete"} + + lineage = perseus.build_sbom_lineage([document], edges=one_edge()) + assert any(edge["from"] == "source:iterator" for edge in lineage["edges"]) + + oversized = [{ + "from": "source:oversized", + "to": "artifact:oversized", + "type": "generates", + "confidence": "high", + "coverage": "complete", + } for _ in range(perseus._SL_MAX_EDGES + 1)] + with pytest.raises(perseus.SBOMLineageError, match="edges|bound"): + perseus.build_sbom_lineage([document], edges=oversized) From 619730c7327f7d19e841d86e268e2115bf9a3ea5 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 21:59:28 +0000 Subject: [PATCH 12/40] test(sbom): cover markerless private source refs --- tests/test_sbom_lineage.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 3a707c5f..ee0bb9b0 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -972,6 +972,12 @@ def test_privacy_sanitizer_covers_fragments_userinfo_markerless_private_refs_and assert "RAW_XML_SUPPLIER" not in json.dumps(xml_document, sort_keys=True) +def test_strict_normalized_source_refs_reject_markerless_private_locators(): + for source_ref in ("artifact:private", "build:home", "source:local", "artifact:raw"): + with pytest.raises(perseus.SBOMLineageError, match="visibility-safe|unsafe credential"): + perseus._sl_strict_source_ref(source_ref) + + def test_spdx_document_id_namespace_and_component_relationship_rebinding_are_consistent(): payload = json.loads(_load("spdx-app.json")) payload["SPDXID"] = "SPDXRef-DOCUMENT-custom" From c594cd7676a6b3e119501e1f39cd437b05846c39 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Wed, 19 Aug 2026 22:23:16 +0000 Subject: [PATCH 13/40] fix(sbom): close privacy namespace and scalar blockers --- perseus.py | 17 +++++++++++++-- src/perseus/sbom_lineage.py | 15 ++++++++++++- tests/test_sbom_lineage.py | 43 +++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/perseus.py b/perseus.py index e8793ac4..8251ba98 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "fcd2773-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "619730c-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44657,6 +44657,15 @@ def _sl_userinfo_locator(value: str) -> bool: continue if re.match(r"^[^/]+:[^/]+", suffix) or "/" in suffix: return True + prefix = value[:at] + namespace_end = prefix.find(":") + if ( + namespace_end >= 0 + and ":" in prefix[namespace_end + 1:] + and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]{0,253}", suffix) + and not re.fullmatch(r"\d+(?:\.\d+){1,3}", suffix) + ): + return True return False @@ -45062,6 +45071,8 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di def _sl_bind_component_alias(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: + if _sl_node_kind(alias) == "document": + return existing = component_id_map.get(alias) if existing is not None and existing != normalized_id: raise SBOMLineageError("ambiguous component alias") @@ -45135,7 +45146,7 @@ def _sl_component( _sl_truncated(component_truncated, "identifiers") for identifier in identifier_values[:_SL_MAX_IDENTIFIERS]: text = _sl_safe_locator(identifier, "component_identifier") - if text: + if text and _sl_node_kind(text) != "document": ids.add(text) if component_id_map is not None: for identifier in ids: @@ -46173,6 +46184,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non if fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") spec_version = document.get("spec_version") + if not isinstance(spec_version, str): + raise SBOMLineageError("normalized SBOM spec_version must be a string") if fmt == "SPDX": _sl_validate_spdx_version(f"SPDX-{spec_version}") else: diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index eeeb1c90..48cf231f 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -221,6 +221,15 @@ def _sl_userinfo_locator(value: str) -> bool: continue if re.match(r"^[^/]+:[^/]+", suffix) or "/" in suffix: return True + prefix = value[:at] + namespace_end = prefix.find(":") + if ( + namespace_end >= 0 + and ":" in prefix[namespace_end + 1:] + and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]{0,253}", suffix) + and not re.fullmatch(r"\d+(?:\.\d+){1,3}", suffix) + ): + return True return False @@ -626,6 +635,8 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di def _sl_bind_component_alias(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: + if _sl_node_kind(alias) == "document": + return existing = component_id_map.get(alias) if existing is not None and existing != normalized_id: raise SBOMLineageError("ambiguous component alias") @@ -699,7 +710,7 @@ def _sl_component( _sl_truncated(component_truncated, "identifiers") for identifier in identifier_values[:_SL_MAX_IDENTIFIERS]: text = _sl_safe_locator(identifier, "component_identifier") - if text: + if text and _sl_node_kind(text) != "document": ids.add(text) if component_id_map is not None: for identifier in ids: @@ -1737,6 +1748,8 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non if fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") spec_version = document.get("spec_version") + if not isinstance(spec_version, str): + raise SBOMLineageError("normalized SBOM spec_version must be a string") if fmt == "SPDX": _sl_validate_spdx_version(f"SPDX-{spec_version}") else: diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index ee0bb9b0..86c5e6a0 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -64,6 +64,18 @@ def test_rejects_unknown_format_and_unsupported_version(): perseus.ingest_sbom_document(json.dumps({"spdxVersion": "SPDX-9.9", "SPDXID": "SPDXRef-DOCUMENT"})) +def test_normalized_spec_version_must_remain_a_supported_string(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:scalar-fixture") + document["spec_version"] = 2.3 + unsigned = dict(document) + unsigned.pop("ingestion_digest") + document["ingestion_digest"] = perseus._sl_ingestion_sha(unsigned, raw) + + with pytest.raises(perseus.SBOMLineageError, match="spec_version.*string|version"): + perseus._sl_validate_document(document, raw_bytes=raw) + + def test_query_returns_auditable_impacted_artifact_path_without_false_clean_claim(): document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:fixture-spdx-json") lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) @@ -126,6 +138,37 @@ def test_credential_bearing_source_and_references_are_not_persisted(): assert document["source_ref"].startswith("sha256:") +def test_terminal_host_userinfo_source_refs_are_hashed_without_hiding_versions(): + raw = _load("spdx-app.json") + private = perseus.ingest_sbom_document(raw, source_ref="artifact:alice:pw@host") + assert private["source_ref"].startswith("sha256:source-ref:") + assert "alice" not in json.dumps(private, sort_keys=True) + + public = perseus.ingest_sbom_document(raw, source_ref="artifact:component@1.2.3") + assert public["source_ref"] == "artifact:component@1.2.3" + assert any( + "pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0" in component["identifiers"] + for component in public["components"] + ) + + +def test_document_namespaces_cannot_be_component_aliases(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [{"type": "purl", "url": "document:unbound"}] + document = perseus.ingest_sbom_document(payload, source_ref="artifact:namespace-fixture") + assert all("document:unbound" not in component["identifiers"] for component in document["components"]) + + valid = perseus.build_sbom_lineage([document], edges=[ + {"from": document["document_id"], "to": "build:alias", "type": "routes", "confidence": "high", "coverage": "complete"}, + ]) + assert any(edge["from"] == document["document_id"] and edge["to"] == "build:alias" for edge in valid["edges"]) + + with pytest.raises(perseus.SBOMLineageError, match="document.*endpoint|bound"): + perseus.build_sbom_lineage([document], edges=[ + {"from": "document:unbound", "to": "build:alias", "type": "routes", "confidence": "high", "coverage": "complete"}, + ]) + + def test_percent_encoded_secrets_are_redacted_from_all_reference_depths(): payload = json.loads(_load("spdx-app.json")) payload["packages"][0]["externalRefs"].append( From 605246fb04c91c0c400396df42c631cf97643297 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 03:17:37 +0000 Subject: [PATCH 14/40] fix: harden SBOM lineage privacy and bounded ingestion --- README.md | 2 +- perseus.py | 216 +++++++++++++++++++++++++++++++++--- src/perseus/sbom_lineage.py | 214 ++++++++++++++++++++++++++++++++--- tests/test_sbom_lineage.py | 153 +++++++++++++++++++++++++ 4 files changed, 553 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index a1211cad..c90895e7 100755 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ Published as [`io.github.Perseus-Computing-LLC/perseus`](https://registry.modelc ### MCP Tools - + MCP tools resolve live state at invocation time, including the canonical Perseus Vault tool. Two additional sensitive tools — `perseus_query` (run a shell command) and `perseus_agent` (execute a local agent subprocess) — are **not** part of this default set: they require explicit `mcp.tool_allowlist` opt-in because they execute commands in the user's local shell (**not sandboxed, full user permissions apply**). diff --git a/perseus.py b/perseus.py index 8251ba98..3f733f9b 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "619730c-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "c594cd7-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44445,6 +44445,7 @@ def cmd_code_map(args, cfg) -> int: """ import hashlib +import ipaddress as _sl_ipaddress import json import math import os as _sl_os @@ -44470,7 +44471,7 @@ def cmd_code_map(args, cfg) -> int: _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") _SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=|authorization\s*[/=:])") -_SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw)(?:[/?:=&_.-]|$)") +_SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw|internal|intranet|corp|localhost)(?:[/?:=&_.-]|$)") _SL_PRIVACY_MARKER_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(api[_-]?key|authorization|password|passwd|secret|token|credential)(?:[/?:=&_.-]|$)") _SL_PUBLIC_PURL_QUERY_KEYS = frozenset({"classifier", "extension", "type", "repository_url"}) _SL_REFERENCE_TYPES = frozenset({ @@ -44548,7 +44549,7 @@ def __setitem__(self, key: str, value: Any) -> None: self.__delitem__(key) entry_size = self._size(value) if entry_size > self._max_bytes: - return + raise SBOMLineageError("provenance receipt exceeds its byte bound") super().__setitem__(key, value) self._entry_bytes[key] = entry_size self._total_bytes += entry_size @@ -44643,6 +44644,39 @@ def _sl_decoded_variants(value: str) -> list[str]: return candidates +def _sl_nonpublic_host(host: str) -> bool: + normalized = host.strip().strip("[]").rstrip(".").casefold() + if not normalized: + return False + try: + return not _sl_ipaddress.ip_address(normalized).is_global + except ValueError: + return normalized in {"localhost", "localhost.localdomain"} or normalized.endswith( + (".invalid", ".local", ".internal", ".intranet", ".lan", ".home", ".test"), + ) + + +def _sl_authority_host(authority: str) -> str: + host = authority.rsplit("@", 1)[-1].strip() + if host.startswith("[") and "]" in host: + return host[1:host.index("]")] + if host.count(":") == 1: + return host.rsplit(":", 1)[0] + return host + + +def _sl_locator_has_nonpublic_host(value: str) -> bool: + for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): + if _sl_nonpublic_host(_sl_authority_host(match.group(1))): + return True + opaque = re.match(r"(?i)^[a-z][a-z0-9+.-]*:([^/?#]*)", value) + if opaque: + scheme = value.split(":", 1)[0].casefold() + if scheme != "pkg" and _sl_nonpublic_host(_sl_authority_host(opaque.group(1))): + return True + return False + + def _sl_userinfo_locator(value: str) -> bool: """Detect URI and git-style userinfo without treating version ``@`` as auth.""" for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): @@ -44659,6 +44693,7 @@ def _sl_userinfo_locator(value: str) -> bool: return True prefix = value[:at] namespace_end = prefix.find(":") + scheme = prefix[:namespace_end].casefold() if namespace_end >= 0 else "" if ( namespace_end >= 0 and ":" in prefix[namespace_end + 1:] @@ -44666,6 +44701,15 @@ def _sl_userinfo_locator(value: str) -> bool: and not re.fullmatch(r"\d+(?:\.\d+){1,3}", suffix) ): return True + opaque_body = prefix[namespace_end + 1:] if namespace_end >= 0 else "" + if ( + namespace_end >= 0 + and scheme != "pkg" + and "/" not in opaque_body + and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]{0,253}", suffix) + and not re.fullmatch(r"\d+(?:\.\d+){0,3}", suffix) + ): + return True return False @@ -44679,13 +44723,23 @@ def _sl_private_locator(value: str) -> bool: or _SL_PRIVATE_LOCATOR_RE.search(decoded) or _SL_PRIVACY_MARKER_RE.search(decoded) or _sl_userinfo_locator(decoded) + or _sl_locator_has_nonpublic_host(decoded) ): return True if "?" in decoded: query = decoded.split("?", 1)[1].split("#", 1)[0] for pair in query.split("&"): - key = pair.split("=", 1)[0].casefold() - if key and key not in _SL_PUBLIC_PURL_QUERY_KEYS: + raw_key, _, raw_value = pair.partition("=") + key = unquote(raw_key).casefold() + query_value = unquote(raw_value) + if lowered.startswith("pkg:"): + if key and key not in _SL_PUBLIC_PURL_QUERY_KEYS: + return True + if ( + _SL_PRIVATE_LOCATOR_RE.search(query_value) + or _SL_PRIVACY_MARKER_RE.search(query_value) + or _sl_locator_has_nonpublic_host(query_value) + ): return True return False @@ -45070,6 +45124,20 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di return result +def _sl_xml_properties(component: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: + container = _sl_single_child(component, "properties") + if container is None: + return [] + nodes = _sl_children(container, "property") + if len(nodes) > _SL_MAX_PROPERTIES: + _sl_truncated(truncated, "properties") + raw_properties = [ + {"name": node.attrib.get("name", ""), "value": node.text or ""} + for node in nodes[:_SL_MAX_PROPERTIES] + ] + return _sl_properties(raw_properties, truncated=truncated) + + def _sl_bind_component_alias(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: if _sl_node_kind(alias) == "document": return @@ -45095,6 +45163,19 @@ def _sl_component_alias_map(documents: list[Mapping[str, Any]]) -> dict[str, str return aliases +def _sl_resolve_component_alias(value: str, aliases: Mapping[str, str]) -> str: + candidates = _sl_decoded_variants(value) + try: + candidates.append(_sl_safe_locator(value, "lineage endpoint")) + except SBOMLineageError: + pass + for candidate in candidates: + resolved = aliases.get(candidate) + if resolved is not None: + return resolved + return value + + def _sl_resolve_external_edge_aliases(edges: list[Any], aliases: Mapping[str, str]) -> list[Any]: resolved: list[Any] = [] for raw in edges: @@ -45106,9 +45187,11 @@ def _sl_resolve_external_edge_aliases(edges: list[Any], aliases: Mapping[str, st if key in raw: item[key] = raw[key] for primary, fallback in (("from", "source"), ("to", "target")): + if primary in item and fallback in item and item[primary] != item[fallback]: + raise SBOMLineageError("conflicting relationship endpoint aliases") key = primary if primary in item else fallback if fallback in item else None if key is not None and isinstance(item.get(key), str): - item[key] = aliases.get(item[key], item[key]) + item[key] = _sl_resolve_component_alias(item[key], aliases) resolved.append(item) return resolved @@ -45648,6 +45731,7 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo _sl_xml_text(ref, "type", default="other"), locator, truncated=component_truncated, )) + refs.extend(_sl_xml_properties(raw, truncated=component_truncated)) licenses = [] license_container = _sl_single_child(raw, "licenses") if license_container is not None: @@ -45802,14 +45886,71 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) -def _sl_bounded_byteslike(source: bytes | bytearray | memoryview) -> bytes: - size = source.nbytes if isinstance(source, memoryview) else len(source) - if size > _SL_MAX_INPUT_BYTES: +def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: + """Reject obviously oversized JSON sources before the encoder copies them.""" + active = set() if active is None else active + if isinstance(value, str): + total += str.__len__(value) + 2 + elif isinstance(value, Mapping): + marker = id(value) + if marker in active: + raise SBOMLineageError("SBOM JSON contains a cycle") + active.add(marker) + total += 2 + for key, item in value.items(): + if not isinstance(key, str): + raise SBOMLineageError("SBOM JSON object keys must be strings") + total += str.__len__(key) + 3 + total = _sl_preflight_json_size(item, total=total, active=active) + if total > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + active.remove(marker) + elif isinstance(value, (list, tuple)): + marker = id(value) + if marker in active: + raise SBOMLineageError("SBOM JSON contains a cycle") + active.add(marker) + total += 2 + for item in value: + total = _sl_preflight_json_size(item, total=total, active=active) + if total > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + active.remove(marker) + elif value is None or isinstance(value, bool): + total += 4 + elif isinstance(value, (int, float)): + total += 1 + if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return total + + +def _sl_bounded_text_bytes(source: str) -> bytes: + if str.__len__(source) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + result = bytearray() + for offset in range(0, str.__len__(source), 64 * 1024): + chunk = str.__getitem__(source, slice(offset, offset + 64 * 1024)).encode("utf-8") + if len(result) + len(chunk) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + result.extend(chunk) + return bytes(result) + + +def _sl_bounded_byteslike(source: bytes | bytearray | memoryview) -> bytes: + view: memoryview | None = None try: - raw_bytes = source if isinstance(source, bytes) else bytes(source) + view = memoryview(source) + if view.nbytes > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + raw_bytes = bytes(view) + except SBOMLineageError: + raise except (BufferError, OverflowError, TypeError, ValueError): raise SBOMLineageError("SBOM bytes-like input is invalid") from None + finally: + if view is not None: + view.release() if len(raw_bytes) > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") return raw_bytes @@ -45847,6 +45988,24 @@ def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allo def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: if re.search(rb" Any: def _sl_payload(document: Any) -> tuple[Any, bytes]: if isinstance(document, Path): return _sl_payload(_sl_read_bounded(document)) - if isinstance(document, (bytes, bytearray, memoryview)): + elif isinstance(document, (bytes, bytearray, memoryview)): raw_bytes = _sl_bounded_byteslike(document) elif isinstance(document, Mapping): try: + _sl_preflight_json_size(document) raw_bytes = _sl_json(document).encode("utf-8") + except SBOMLineageError: + raise except (TypeError, ValueError) as exc: raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc elif isinstance(document, str): possible_path = Path(document) if "\n" not in document and possible_path.exists(): return _sl_payload(possible_path) - raw_bytes = document.encode("utf-8") + raw_bytes = _sl_bounded_text_bytes(document) else: raise SBOMLineageError("SBOM input must be bytes, text, path, or object") if len(raw_bytes) > _SL_MAX_INPUT_BYTES: @@ -46287,6 +46449,13 @@ def _sl_bounded_external_edges(edges: Any) -> list[Any]: return result +def _sl_account_input_bytes(total: int, raw_bytes: bytes) -> int: + total += len(raw_bytes) + if total > _SL_MAX_CLI_TOTAL_BYTES: + raise SBOMLineageError("SBOM input exceeds the aggregate byte bound") + return total + + def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: if edges is None: return [] @@ -46343,7 +46512,12 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any if raw_documents is not None: if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match the document list within its bound") - rebound_raw = list(raw_documents) + rebound_raw = [] + raw_total = 0 + for raw_document in raw_documents: + _, raw_bytes = _sl_payload(raw_document) + raw_total = _sl_account_input_bytes(raw_total, raw_bytes) + rebound_raw.append(raw_bytes) truncated: list[str] = [] if len(documents) > _SL_MAX_DOCUMENTS: truncated.append("documents") @@ -47001,17 +47175,21 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: ("edges", [edge_path] if edge_path is not None else None), )) documents = [] + actual_input_bytes = 0 for path in document_paths or []: payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) raw_inputs = None if raw_document_paths is not None: raw_inputs = [] for raw_path in raw_document_paths: _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) raw_inputs.append(raw_bytes) if edge_path is not None: - edges, _ = _sl_load_json_bounded(Path(edge_path)) + edges, edge_bytes = _sl_load_json_bounded(Path(edge_path)) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, edge_bytes) if not isinstance(edges, list): raise SBOMLineageError("lineage edges JSON must be a list") else: @@ -47022,7 +47200,9 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage_path = _sl_cli_checked_path(getattr(args, "lineage", None), "lineage") _sl_cli_preflight_paths((("lineage", [lineage_path]), ("raw_documents", raw_document_paths))) - lineage, _ = _sl_load_json_bounded(Path(lineage_path)) + actual_input_bytes = 0 + lineage, lineage_bytes = _sl_load_json_bounded(Path(lineage_path)) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, lineage_bytes) if not isinstance(lineage, Mapping): raise SBOMLineageError("lineage JSON root must be an object") raw_inputs = None @@ -47030,7 +47210,11 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: lineage_documents = lineage.get("documents") if not isinstance(lineage_documents, list) or len(raw_document_paths) != len(lineage_documents): raise SBOMLineageError("raw_documents must contain one raw source per lineage document") - raw_inputs = [_sl_load_json_bounded(Path(path), allow_non_json=True)[1] for path in raw_document_paths] + raw_inputs = [] + for path in raw_document_paths: + _, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) + raw_inputs.append(raw_bytes) document = query_sbom_lineage( lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, ) diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 48cf231f..ea81927a 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -9,6 +9,7 @@ from __future__ import annotations import hashlib +import ipaddress as _sl_ipaddress import json import math import os as _sl_os @@ -34,7 +35,7 @@ _SL_CDX_NAMESPACE_RE = re.compile(r"bom-(1\.\d+)\.xsd") _SL_PUBLIC_SOURCE_RE = re.compile(r"^(?:file|artifact|vault|ledger|build|deployment):[A-Za-z0-9][A-Za-z0-9_.:/#@+%~\-]{0,255}$") _SL_SENSITIVE_REFERENCE_RE = re.compile(r"(?i)(?:bearer(?:\s+|\s*:)|basic(?:\s+|\s*:)|password\s*=|passwd\s*=|secret\s*=|token\s*=|api[_-]?key\s*[/=:]|credential\s*=|authorization\s*[/=:])") -_SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw)(?:[/?:=&_.-]|$)") +_SL_PRIVATE_LOCATOR_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(private|home|user|local|raw|internal|intranet|corp|localhost)(?:[/?:=&_.-]|$)") _SL_PRIVACY_MARKER_RE = re.compile(r"(?i)(?:^|[/?:=&_.-])(api[_-]?key|authorization|password|passwd|secret|token|credential)(?:[/?:=&_.-]|$)") _SL_PUBLIC_PURL_QUERY_KEYS = frozenset({"classifier", "extension", "type", "repository_url"}) _SL_REFERENCE_TYPES = frozenset({ @@ -112,7 +113,7 @@ def __setitem__(self, key: str, value: Any) -> None: self.__delitem__(key) entry_size = self._size(value) if entry_size > self._max_bytes: - return + raise SBOMLineageError("provenance receipt exceeds its byte bound") super().__setitem__(key, value) self._entry_bytes[key] = entry_size self._total_bytes += entry_size @@ -207,6 +208,39 @@ def _sl_decoded_variants(value: str) -> list[str]: return candidates +def _sl_nonpublic_host(host: str) -> bool: + normalized = host.strip().strip("[]").rstrip(".").casefold() + if not normalized: + return False + try: + return not _sl_ipaddress.ip_address(normalized).is_global + except ValueError: + return normalized in {"localhost", "localhost.localdomain"} or normalized.endswith( + (".invalid", ".local", ".internal", ".intranet", ".lan", ".home", ".test"), + ) + + +def _sl_authority_host(authority: str) -> str: + host = authority.rsplit("@", 1)[-1].strip() + if host.startswith("[") and "]" in host: + return host[1:host.index("]")] + if host.count(":") == 1: + return host.rsplit(":", 1)[0] + return host + + +def _sl_locator_has_nonpublic_host(value: str) -> bool: + for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): + if _sl_nonpublic_host(_sl_authority_host(match.group(1))): + return True + opaque = re.match(r"(?i)^[a-z][a-z0-9+.-]*:([^/?#]*)", value) + if opaque: + scheme = value.split(":", 1)[0].casefold() + if scheme != "pkg" and _sl_nonpublic_host(_sl_authority_host(opaque.group(1))): + return True + return False + + def _sl_userinfo_locator(value: str) -> bool: """Detect URI and git-style userinfo without treating version ``@`` as auth.""" for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): @@ -223,6 +257,7 @@ def _sl_userinfo_locator(value: str) -> bool: return True prefix = value[:at] namespace_end = prefix.find(":") + scheme = prefix[:namespace_end].casefold() if namespace_end >= 0 else "" if ( namespace_end >= 0 and ":" in prefix[namespace_end + 1:] @@ -230,6 +265,15 @@ def _sl_userinfo_locator(value: str) -> bool: and not re.fullmatch(r"\d+(?:\.\d+){1,3}", suffix) ): return True + opaque_body = prefix[namespace_end + 1:] if namespace_end >= 0 else "" + if ( + namespace_end >= 0 + and scheme != "pkg" + and "/" not in opaque_body + and re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9.-]{0,253}", suffix) + and not re.fullmatch(r"\d+(?:\.\d+){0,3}", suffix) + ): + return True return False @@ -243,13 +287,23 @@ def _sl_private_locator(value: str) -> bool: or _SL_PRIVATE_LOCATOR_RE.search(decoded) or _SL_PRIVACY_MARKER_RE.search(decoded) or _sl_userinfo_locator(decoded) + or _sl_locator_has_nonpublic_host(decoded) ): return True if "?" in decoded: query = decoded.split("?", 1)[1].split("#", 1)[0] for pair in query.split("&"): - key = pair.split("=", 1)[0].casefold() - if key and key not in _SL_PUBLIC_PURL_QUERY_KEYS: + raw_key, _, raw_value = pair.partition("=") + key = unquote(raw_key).casefold() + query_value = unquote(raw_value) + if lowered.startswith("pkg:"): + if key and key not in _SL_PUBLIC_PURL_QUERY_KEYS: + return True + if ( + _SL_PRIVATE_LOCATOR_RE.search(query_value) + or _SL_PRIVACY_MARKER_RE.search(query_value) + or _sl_locator_has_nonpublic_host(query_value) + ): return True return False @@ -634,6 +688,20 @@ def _sl_properties(value: Any, *, truncated: list[str] | None = None) -> list[di return result +def _sl_xml_properties(component: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: + container = _sl_single_child(component, "properties") + if container is None: + return [] + nodes = _sl_children(container, "property") + if len(nodes) > _SL_MAX_PROPERTIES: + _sl_truncated(truncated, "properties") + raw_properties = [ + {"name": node.attrib.get("name", ""), "value": node.text or ""} + for node in nodes[:_SL_MAX_PROPERTIES] + ] + return _sl_properties(raw_properties, truncated=truncated) + + def _sl_bind_component_alias(component_id_map: dict[str, str], alias: str, normalized_id: str) -> None: if _sl_node_kind(alias) == "document": return @@ -659,6 +727,19 @@ def _sl_component_alias_map(documents: list[Mapping[str, Any]]) -> dict[str, str return aliases +def _sl_resolve_component_alias(value: str, aliases: Mapping[str, str]) -> str: + candidates = _sl_decoded_variants(value) + try: + candidates.append(_sl_safe_locator(value, "lineage endpoint")) + except SBOMLineageError: + pass + for candidate in candidates: + resolved = aliases.get(candidate) + if resolved is not None: + return resolved + return value + + def _sl_resolve_external_edge_aliases(edges: list[Any], aliases: Mapping[str, str]) -> list[Any]: resolved: list[Any] = [] for raw in edges: @@ -670,9 +751,11 @@ def _sl_resolve_external_edge_aliases(edges: list[Any], aliases: Mapping[str, st if key in raw: item[key] = raw[key] for primary, fallback in (("from", "source"), ("to", "target")): + if primary in item and fallback in item and item[primary] != item[fallback]: + raise SBOMLineageError("conflicting relationship endpoint aliases") key = primary if primary in item else fallback if fallback in item else None if key is not None and isinstance(item.get(key), str): - item[key] = aliases.get(item[key], item[key]) + item[key] = _sl_resolve_component_alias(item[key], aliases) resolved.append(item) return resolved @@ -1212,6 +1295,7 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo _sl_xml_text(ref, "type", default="other"), locator, truncated=component_truncated, )) + refs.extend(_sl_xml_properties(raw, truncated=component_truncated)) licenses = [] license_container = _sl_single_child(raw, "licenses") if license_container is not None: @@ -1366,14 +1450,71 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) -def _sl_bounded_byteslike(source: bytes | bytearray | memoryview) -> bytes: - size = source.nbytes if isinstance(source, memoryview) else len(source) - if size > _SL_MAX_INPUT_BYTES: +def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: + """Reject obviously oversized JSON sources before the encoder copies them.""" + active = set() if active is None else active + if isinstance(value, str): + total += str.__len__(value) + 2 + elif isinstance(value, Mapping): + marker = id(value) + if marker in active: + raise SBOMLineageError("SBOM JSON contains a cycle") + active.add(marker) + total += 2 + for key, item in value.items(): + if not isinstance(key, str): + raise SBOMLineageError("SBOM JSON object keys must be strings") + total += str.__len__(key) + 3 + total = _sl_preflight_json_size(item, total=total, active=active) + if total > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + active.remove(marker) + elif isinstance(value, (list, tuple)): + marker = id(value) + if marker in active: + raise SBOMLineageError("SBOM JSON contains a cycle") + active.add(marker) + total += 2 + for item in value: + total = _sl_preflight_json_size(item, total=total, active=active) + if total > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + active.remove(marker) + elif value is None or isinstance(value, bool): + total += 4 + elif isinstance(value, (int, float)): + total += 1 + if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return total + + +def _sl_bounded_text_bytes(source: str) -> bytes: + if str.__len__(source) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + result = bytearray() + for offset in range(0, str.__len__(source), 64 * 1024): + chunk = str.__getitem__(source, slice(offset, offset + 64 * 1024)).encode("utf-8") + if len(result) + len(chunk) > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + result.extend(chunk) + return bytes(result) + + +def _sl_bounded_byteslike(source: bytes | bytearray | memoryview) -> bytes: + view: memoryview | None = None try: - raw_bytes = source if isinstance(source, bytes) else bytes(source) + view = memoryview(source) + if view.nbytes > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + raw_bytes = bytes(view) + except SBOMLineageError: + raise except (BufferError, OverflowError, TypeError, ValueError): raise SBOMLineageError("SBOM bytes-like input is invalid") from None + finally: + if view is not None: + view.release() if len(raw_bytes) > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") return raw_bytes @@ -1411,6 +1552,24 @@ def _sl_load_json_bounded(source: Path | bytes | bytearray | memoryview, *, allo def _sl_parse_xml_bounded(raw_bytes: bytes) -> Any: if re.search(rb" Any: def _sl_payload(document: Any) -> tuple[Any, bytes]: if isinstance(document, Path): return _sl_payload(_sl_read_bounded(document)) - if isinstance(document, (bytes, bytearray, memoryview)): + elif isinstance(document, (bytes, bytearray, memoryview)): raw_bytes = _sl_bounded_byteslike(document) elif isinstance(document, Mapping): try: + _sl_preflight_json_size(document) raw_bytes = _sl_json(document).encode("utf-8") + except SBOMLineageError: + raise except (TypeError, ValueError) as exc: raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc elif isinstance(document, str): possible_path = Path(document) if "\n" not in document and possible_path.exists(): return _sl_payload(possible_path) - raw_bytes = document.encode("utf-8") + raw_bytes = _sl_bounded_text_bytes(document) else: raise SBOMLineageError("SBOM input must be bytes, text, path, or object") if len(raw_bytes) > _SL_MAX_INPUT_BYTES: @@ -1851,6 +2013,13 @@ def _sl_bounded_external_edges(edges: Any) -> list[Any]: return result +def _sl_account_input_bytes(total: int, raw_bytes: bytes) -> int: + total += len(raw_bytes) + if total > _SL_MAX_CLI_TOTAL_BYTES: + raise SBOMLineageError("SBOM input exceeds the aggregate byte bound") + return total + + def _sl_lineage_edges(edges: Any, *, truncated: list[str] | None = None) -> list[dict[str, Any]]: if edges is None: return [] @@ -1907,7 +2076,12 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any if raw_documents is not None: if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match the document list within its bound") - rebound_raw = list(raw_documents) + rebound_raw = [] + raw_total = 0 + for raw_document in raw_documents: + _, raw_bytes = _sl_payload(raw_document) + raw_total = _sl_account_input_bytes(raw_total, raw_bytes) + rebound_raw.append(raw_bytes) truncated: list[str] = [] if len(documents) > _SL_MAX_DOCUMENTS: truncated.append("documents") @@ -2565,17 +2739,21 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: ("edges", [edge_path] if edge_path is not None else None), )) documents = [] + actual_input_bytes = 0 for path in document_paths or []: payload, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) documents.append(payload if isinstance(payload, Mapping) and payload.get("schema_version") == _SL_SCHEMA else ingest_sbom_document(raw_bytes, source_ref=f"file:{path}")) raw_inputs = None if raw_document_paths is not None: raw_inputs = [] for raw_path in raw_document_paths: _, raw_bytes = _sl_load_json_bounded(Path(raw_path), allow_non_json=True) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) raw_inputs.append(raw_bytes) if edge_path is not None: - edges, _ = _sl_load_json_bounded(Path(edge_path)) + edges, edge_bytes = _sl_load_json_bounded(Path(edge_path)) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, edge_bytes) if not isinstance(edges, list): raise SBOMLineageError("lineage edges JSON must be a list") else: @@ -2586,7 +2764,9 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage_path = _sl_cli_checked_path(getattr(args, "lineage", None), "lineage") _sl_cli_preflight_paths((("lineage", [lineage_path]), ("raw_documents", raw_document_paths))) - lineage, _ = _sl_load_json_bounded(Path(lineage_path)) + actual_input_bytes = 0 + lineage, lineage_bytes = _sl_load_json_bounded(Path(lineage_path)) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, lineage_bytes) if not isinstance(lineage, Mapping): raise SBOMLineageError("lineage JSON root must be an object") raw_inputs = None @@ -2594,7 +2774,11 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: lineage_documents = lineage.get("documents") if not isinstance(lineage_documents, list) or len(raw_document_paths) != len(lineage_documents): raise SBOMLineageError("raw_documents must contain one raw source per lineage document") - raw_inputs = [_sl_load_json_bounded(Path(path), allow_non_json=True)[1] for path in raw_document_paths] + raw_inputs = [] + for path in raw_document_paths: + _, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) + raw_inputs.append(raw_bytes) document = query_sbom_lineage( lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, ) diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 86c5e6a0..74aa35c3 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1534,3 +1534,156 @@ def one_edge(): } for _ in range(perseus._SL_MAX_EDGES + 1)] with pytest.raises(perseus.SBOMLineageError, match="edges|bound"): perseus.build_sbom_lineage([document], edges=oversized) + + +def test_host_only_userinfo_and_markerless_private_locators_are_not_persisted(): + raw = _load("spdx-app.json") + for source_ref in ("artifact:alice@host", "artifact:alice%40host", "git:alice@host"): + document = perseus.ingest_sbom_document(raw, source_ref=source_ref) + assert document["source_ref"].startswith("sha256:source-ref:") + assert "alice" not in json.dumps(document, sort_keys=True) + + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["purl"] = "pkg:generic/foo@1?repository_url=https://10.0.0.1/repo" + component["externalReferences"] = [ + {"type": "website", "url": "https://10.0.0.1/repo"}, + {"type": "website", "url": "artifact:10.0.0.1/repo"}, + {"type": "website", "url": "https://example.invalid/?classifier=ALPHA_INTERNAL"}, + ] + document = perseus.ingest_sbom_document(payload) + serialized = json.dumps(document, sort_keys=True) + assert "10.0.0.1" not in serialized + assert "ALPHA_INTERNAL" not in serialized + + +def test_mapping_and_text_input_bounds_are_checked_before_serialization(monkeypatch): + monkeypatch.setattr(perseus, "_SL_MAX_INPUT_BYTES", 16) + called = [] + + def unexpected_json(value): + called.append(value) + raise AssertionError("oversized mapping was serialized before its bound was checked") + + monkeypatch.setattr(perseus, "_sl_json", unexpected_json) + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_payload({"bomFormat": "CycloneDX", "ignored": "x" * 32}) + assert called == [] + + class ExplodingText(str): + def encode(self, *args, **kwargs): + raise AssertionError("oversized text was encoded before its bound was checked") + + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_payload(ExplodingText("x" * 32)) + + +def test_hostile_bytearray_subclass_is_bounded_from_buffer_before_copy(monkeypatch): + monkeypatch.setattr(perseus, "_SL_MAX_INPUT_BYTES", 2) + + class LyingBytearray(bytearray): + def __len__(self): + return 0 + + def __bytes__(self): + raise AssertionError("oversized bytearray was copied before its bound was checked") + + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_bounded_byteslike(LyingBytearray(b"{} ")) + + +def test_utf16_xml_dtd_and_entities_are_rejected(): + xml = ']>' + with pytest.raises(perseus.SBOMLineageError, match="DTD|entity"): + perseus.ingest_sbom_document(xml.encode("utf-16")) + + +def test_actual_loaded_cli_buffers_and_direct_raw_rebinding_obey_aggregate_bound(tmp_path, monkeypatch, capsys): + raw = _load("spdx-app.json") + second_payload = json.loads(raw) + second_payload["SPDXID"] = "SPDXRef-DOCUMENT-SECOND" + second_payload["relationships"][0]["spdxElementId"] = "SPDXRef-DOCUMENT-SECOND" + second_raw = json.dumps(second_payload, separators=(",", ":")).encode("utf-8") + first = tmp_path / "first.json" + second = tmp_path / "second.json" + first.write_bytes(raw) + second.write_bytes(second_raw) + original = perseus._sl_load_json_bounded + + def inflated(path, **kwargs): + value, raw_bytes = original(path, **kwargs) + return value, raw_bytes + (b" " * len(raw_bytes)) + + monkeypatch.setattr(perseus, "_sl_load_json_bounded", inflated) + monkeypatch.setattr(perseus, "_SL_MAX_CLI_TOTAL_BYTES", len(raw) * 2) + args = type("Args", (), { + "sbom_command": "merge", "documents": [str(first), str(second)], + "raw_documents": None, "edges": None, "output": None, "json": True, + })() + assert perseus.cmd_sbom(args, {}) == 1 + assert "aggregate" in json.loads(capsys.readouterr().out)["error"] + + documents = [ + perseus.ingest_sbom_document(raw, source_ref=f"artifact:aggregate-{index}") + for index in range(2) + ] + monkeypatch.setattr(perseus, "_SL_MAX_CLI_TOTAL_BYTES", len(raw) * 2 - 1) + with pytest.raises(perseus.SBOMLineageError, match="aggregate|bytes"): + perseus.build_sbom_lineage(documents, edges=[], raw_documents=[raw, raw]) + + +def test_cyclonedx_xml_properties_are_retained_in_complete_coverage(): + needle = ( + " pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0\n" + " Apache-2.0" + ) + replacement = ( + " pkg:maven/org.apache.logging.log4j/log4j-core@2.17.0\n" + " CVE-XML-PROPERTY\n" + " Apache-2.0" + ) + xml = _load("cyclonedx-app.xml").decode("utf-8").replace( + "2026-08-19T12:00:00Z", + "2026-08-19T12:00:00Zperseus-build-xml", + ) + xml = xml.replace(needle, replacement) + document = perseus.ingest_sbom_document(xml) + component = next(item for item in document["components"] if item["name"] == "log4j-core") + assert any(reference["type"] == "vex" for reference in component["references"]) + assert document["coverage"]["state"] == "complete" + + +def test_sensitive_purl_external_edges_bind_to_the_component_not_a_stub(): + payload = json.loads(_load("cyclonedx-app.json")) + component = payload["components"][0] + component["bom-ref"] = "custom:component" + component["purl"] = "pkg:generic/foo@1?token=RAWSECRET" + document = perseus.ingest_sbom_document(payload) + raw_purl = component["purl"] + lineage = perseus.build_sbom_lineage([document], edges=[ + {"from": raw_purl, "to": "build:bound", "type": "built_into", "confidence": "high", "coverage": "complete"}, + ]) + edge = next(item for item in lineage["edges"] if item["to"] == "build:bound") + assert edge["from"] == "custom:component" + assert not any(node["node_id"].startswith("sha256:") for node in lineage["nodes"]) + + +def test_external_edges_reject_conflicting_endpoint_alias_fields(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:edge-alias") + with pytest.raises(perseus.SBOMLineageError, match="conflicting|endpoint|alias"): + perseus.build_sbom_lineage([document], edges=[ + {"from": "source:good", "source": "source:bad", "to": "artifact:x", "target": "artifact:x", "type": "generates", "confidence": "high", "coverage": "complete"}, + ]) + + +def test_oversized_lineage_receipts_fail_closed_instead_of_being_dropped(): + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:receipt-bound") + previous = perseus._SL_LINEAGE_PROVENANCE._max_bytes + perseus._SL_LINEAGE_PROVENANCE.clear() + perseus._SL_LINEAGE_PROVENANCE._max_bytes = 1 + try: + with pytest.raises(perseus.SBOMLineageError, match="provenance|receipt|bound"): + perseus.build_sbom_lineage([document], edges=[]) + finally: + perseus._SL_LINEAGE_PROVENANCE._max_bytes = previous + perseus._SL_LINEAGE_PROVENANCE.clear() From 6109831658fc169261be8458471aa40ea5b88d27 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 03:18:01 +0000 Subject: [PATCH 15/40] build: refresh SBOM lineage provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 3f733f9b..985fbb6d 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "c594cd7-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "605246f" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 57f3c614d5d7b528eae36bcc32b1fcd5c9ab1892 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 03:23:02 +0000 Subject: [PATCH 16/40] fix: reject escaped oversized SBOM mappings before encoding --- perseus.py | 37 ++++++++++++++++++++++++++++++------- src/perseus/sbom_lineage.py | 35 +++++++++++++++++++++++++++++------ tests/test_sbom_lineage.py | 3 +++ 3 files changed, 62 insertions(+), 13 deletions(-) diff --git a/perseus.py b/perseus.py index 985fbb6d..695278c9 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "605246f" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "6109831-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -45886,21 +45886,38 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) +def _sl_json_string_size(value: str) -> int: + size = 2 + for index in range(str.__len__(value)): + codepoint = ord(str.__getitem__(value, index)) + if codepoint in (0x22, 0x5C): + size += 2 + elif codepoint <= 0x1F: + size += 2 if codepoint in (0x08, 0x09, 0x0A, 0x0C, 0x0D) else 6 + elif 0x20 <= codepoint <= 0x7E: + size += 1 + elif codepoint > 0xFFFF: + size += 12 + else: + size += 6 + return size + + def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: """Reject obviously oversized JSON sources before the encoder copies them.""" active = set() if active is None else active if isinstance(value, str): - total += str.__len__(value) + 2 + total += _sl_json_string_size(value) elif isinstance(value, Mapping): marker = id(value) if marker in active: raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) total += 2 - for key, item in value.items(): + for index, (key, item) in enumerate(value.items()): if not isinstance(key, str): raise SBOMLineageError("SBOM JSON object keys must be strings") - total += str.__len__(key) + 3 + total += (1 if index else 0) + _sl_json_string_size(key) + 1 total = _sl_preflight_json_size(item, total=total, active=active) if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") @@ -45911,15 +45928,21 @@ def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | No raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) total += 2 - for item in value: + for index, item in enumerate(value): + total += 1 if index else 0 total = _sl_preflight_json_size(item, total=total, active=active) if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") active.remove(marker) - elif value is None or isinstance(value, bool): + elif value is None: total += 4 + elif isinstance(value, bool): + total += 4 if value else 5 elif isinstance(value, (int, float)): - total += 1 + try: + total += len(str(value)) + except (ValueError, OverflowError): + raise SBOMLineageError("SBOM JSON number is invalid") from None if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") return total diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index ea81927a..3e221bd7 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -1450,21 +1450,38 @@ def _sl_validate_json_values(value: Any) -> None: stack.extend(current) +def _sl_json_string_size(value: str) -> int: + size = 2 + for index in range(str.__len__(value)): + codepoint = ord(str.__getitem__(value, index)) + if codepoint in (0x22, 0x5C): + size += 2 + elif codepoint <= 0x1F: + size += 2 if codepoint in (0x08, 0x09, 0x0A, 0x0C, 0x0D) else 6 + elif 0x20 <= codepoint <= 0x7E: + size += 1 + elif codepoint > 0xFFFF: + size += 12 + else: + size += 6 + return size + + def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: """Reject obviously oversized JSON sources before the encoder copies them.""" active = set() if active is None else active if isinstance(value, str): - total += str.__len__(value) + 2 + total += _sl_json_string_size(value) elif isinstance(value, Mapping): marker = id(value) if marker in active: raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) total += 2 - for key, item in value.items(): + for index, (key, item) in enumerate(value.items()): if not isinstance(key, str): raise SBOMLineageError("SBOM JSON object keys must be strings") - total += str.__len__(key) + 3 + total += (1 if index else 0) + _sl_json_string_size(key) + 1 total = _sl_preflight_json_size(item, total=total, active=active) if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") @@ -1475,15 +1492,21 @@ def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | No raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) total += 2 - for item in value: + for index, item in enumerate(value): + total += 1 if index else 0 total = _sl_preflight_json_size(item, total=total, active=active) if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") active.remove(marker) - elif value is None or isinstance(value, bool): + elif value is None: total += 4 + elif isinstance(value, bool): + total += 4 if value else 5 elif isinstance(value, (int, float)): - total += 1 + try: + total += len(str(value)) + except (ValueError, OverflowError): + raise SBOMLineageError("SBOM JSON number is invalid") from None if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") return total diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 74aa35c3..eefcbb7c 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1569,6 +1569,9 @@ def unexpected_json(value): with pytest.raises(perseus.SBOMLineageError, match="bytes"): perseus._sl_payload({"bomFormat": "CycloneDX", "ignored": "x" * 32}) assert called == [] + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_payload({"x": "\\" * 8}) + assert len(called) == 0 class ExplodingText(str): def encode(self, *args, **kwargs): From a2820ace0d503b0014c24930f2afcf2029091823 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 03:23:07 +0000 Subject: [PATCH 17/40] build: refresh final SBOM provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 695278c9..2235f0e9 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "6109831-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "57f3c61" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 92889c81530fd939247c4f0010b005fa3ef16cf6 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 03:53:55 +0000 Subject: [PATCH 18/40] fix: close remaining SBOM lineage review blockers --- perseus.py | 163 +++++++++++++++++++++++++----------- src/perseus/sbom_lineage.py | 161 ++++++++++++++++++++++++----------- tests/test_sbom_lineage.py | 125 +++++++++++++++++++++++++++ 3 files changed, 352 insertions(+), 97 deletions(-) diff --git a/perseus.py b/perseus.py index 2235f0e9..294cb50c 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "57f3c61" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "a2820ac-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44677,6 +44677,16 @@ def _sl_locator_has_nonpublic_host(value: str) -> bool: return False +def _sl_query_value_has_nonpublic_host(value: str) -> bool: + candidate = value.strip() + if _sl_nonpublic_host(candidate): + return True + return any( + _sl_nonpublic_host(match.group(0)) + for match in re.finditer(r"(? bool: """Detect URI and git-style userinfo without treating version ``@`` as auth.""" for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): @@ -44738,7 +44748,7 @@ def _sl_private_locator(value: str) -> bool: if ( _SL_PRIVATE_LOCATOR_RE.search(query_value) or _SL_PRIVACY_MARKER_RE.search(query_value) - or _sl_locator_has_nonpublic_host(query_value) + or _sl_query_value_has_nonpublic_host(query_value) ): return True return False @@ -44828,10 +44838,10 @@ def _sl_safe_locator(value: Any, field: str) -> str: def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: - if value == "": - return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" if not isinstance(value, str): raise SBOMLineageError("source_ref must be a string") + if value == "": + return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" text = _sl_text(value, "source_ref", required=True, limit=512) raw_digest = hashlib.sha256(raw_bytes).hexdigest() if text.startswith("sha256:source-ref:"): @@ -44940,12 +44950,17 @@ def _sl_xml_scalar(element: Any, names: tuple[str, ...], *, default: str = "") - if _sl_local(child).casefold() not in wanted: continue text = (child.text or "").strip() if child.text else "" - scalar = text - for key, value in child.attrib.items(): - if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: - scalar = str(value).lstrip("#") - break - values.append(scalar) + identity_values = [ + str(value).lstrip("#") + for key, value in child.attrib.items() + if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value + ] + if len(set(identity_values)) > 1: + raise SBOMLineageError("XML identity attributes conflict") + identity = identity_values[0] if identity_values else "" + if identity and text and identity != text: + raise SBOMLineageError("XML identity attribute conflicts with element text") + values.append(identity or text) if len(values) > 1: raise SBOMLineageError("XML singleton element is duplicated") return values[0] if values else default @@ -45557,7 +45572,12 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s creators = metadata.get("authors", []) if not isinstance(creators, list): raise SBOMLineageError("metadata.authors must be a list") - supplier = _sl_supplier(creators if creators else (metadata_component or {})) + if creators: + supplier = _sl_supplier(creators) + elif isinstance(metadata_component, Mapping): + supplier = _sl_supplier(metadata_component.get("supplier")) + else: + supplier = "" if "serialNumber" in value: document_id = _sl_id(value.get("serialNumber"), "serialNumber") else: @@ -45759,6 +45779,9 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if not match: raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") version = _sl_validate_cdx_version(match.group(1)) + expected_namespace = f"http://cyclonedx.org/schema/bom-{version}.xsd" + if namespace.casefold() != expected_namespace: + raise SBOMLineageError("CycloneDX XML namespace is not authoritative") metadata = _sl_single_child(root, "metadata") metadata_component = _sl_single_child(metadata, "component") if metadata is not None else None raw_components = [] @@ -45803,10 +45826,12 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" authors = _sl_single_child(metadata, "authors") if metadata is not None else None author_nodes = _sl_children(authors, "author") if authors is not None else [] - author = _sl_supplier([_sl_xml_text(item, "name") for item in author_nodes]) if author_nodes else "" + component_supplier_node = _sl_single_child(metadata_component, "supplier") if metadata_component is not None else None + component_supplier = _sl_xml_text(component_supplier_node, "name") if component_supplier_node is not None else "" + document_supplier = _sl_supplier([_sl_xml_text(item, "name") for item in author_nodes]) if author_nodes else _sl_supplier(component_supplier) return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, - document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, + document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=document_supplier, components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, metadata_component_id=components[0]["component_id"] if metadata_component is not None else "", ) @@ -45903,46 +45928,75 @@ def _sl_json_string_size(value: str) -> int: return size -def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: - """Reject obviously oversized JSON sources before the encoder copies them.""" +def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[Any, int]: + """Snapshot one JSON-compatible value while accounting for its exact JSON size.""" active = set() if active is None else active if isinstance(value, str): - total += _sl_json_string_size(value) - elif isinstance(value, Mapping): + text = str.__str__(value) + size = _sl_json_string_size(text) + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return text, size + if isinstance(value, Mapping): marker = id(value) if marker in active: raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) - total += 2 - for index, (key, item) in enumerate(value.items()): - if not isinstance(key, str): - raise SBOMLineageError("SBOM JSON object keys must be strings") - total += (1 if index else 0) + _sl_json_string_size(key) + 1 - total = _sl_preflight_json_size(item, total=total, active=active) - if total > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") - active.remove(marker) - elif isinstance(value, (list, tuple)): + snapshot: dict[str, Any] = {} + size = 2 + try: + for index, (key, item) in enumerate(value.items()): + if not isinstance(key, str): + raise SBOMLineageError("SBOM JSON object keys must be strings") + key_text = str.__str__(key) + if key_text in snapshot: + raise SBOMLineageError("duplicate JSON object key") + child, child_size = _sl_snapshot_json(item, active=active) + size += (1 if index else 0) + _sl_json_string_size(key_text) + 1 + child_size + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + snapshot[key_text] = child + finally: + active.remove(marker) + return snapshot, size + if isinstance(value, (list, tuple)): marker = id(value) if marker in active: raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) - total += 2 - for index, item in enumerate(value): - total += 1 if index else 0 - total = _sl_preflight_json_size(item, total=total, active=active) - if total > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") - active.remove(marker) - elif value is None: - total += 4 - elif isinstance(value, bool): - total += 4 if value else 5 - elif isinstance(value, (int, float)): + snapshot_list: list[Any] = [] + size = 2 try: - total += len(str(value)) + for index, item in enumerate(value): + child, child_size = _sl_snapshot_json(item, active=active) + size += (1 if index else 0) + child_size + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + snapshot_list.append(child) + finally: + active.remove(marker) + return snapshot_list, size + if value is None: + return None, 4 + if isinstance(value, bool): + return value, 4 if value else 5 + if isinstance(value, (int, float)): + if isinstance(value, float) and not math.isfinite(value): + raise SBOMLineageError("non-finite JSON number is not allowed") + try: + size = len(str(value)) except (ValueError, OverflowError): raise SBOMLineageError("SBOM JSON number is invalid") from None + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return value, size + raise SBOMLineageError("SBOM JSON contains an unsupported value") + + +def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: + """Compatibility wrapper returning the exact size of a bounded JSON snapshot.""" + _, size = _sl_snapshot_json(value, active=active) + total += size if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") return total @@ -46065,17 +46119,18 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_bounded_byteslike(document) elif isinstance(document, Mapping): try: - _sl_preflight_json_size(document) - raw_bytes = _sl_json(document).encode("utf-8") + snapshot, _ = _sl_snapshot_json(document) + raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise except (TypeError, ValueError) as exc: raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc elif isinstance(document, str): + bounded_text = _sl_bounded_text_bytes(document) possible_path = Path(document) if "\n" not in document and possible_path.exists(): return _sl_payload(possible_path) - raw_bytes = _sl_bounded_text_bytes(document) + raw_bytes = bounded_text else: raise SBOMLineageError("SBOM input must be bytes, text, path, or object") if len(raw_bytes) > _SL_MAX_INPUT_BYTES: @@ -46293,7 +46348,12 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") raise SBOMLineageError(f"{field} direction is not allowed") if rel_type != rel_type.casefold().replace(" ", "_"): raise SBOMLineageError(f"{field}.type is not normalized") - if confidence not in _SL_CONFIDENCE or coverage not in _SL_COVERAGE: + if ( + not isinstance(confidence, str) + or not isinstance(coverage, str) + or confidence not in _SL_CONFIDENCE + or coverage not in _SL_COVERAGE + ): raise SBOMLineageError(f"{field} confidence or coverage is invalid") result: dict[str, Any] = { "from": source, "to": target, "type": rel_type, @@ -46406,7 +46466,7 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non _sl_reject_unbound_document_edges(checked_relationships, document_id) coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") - if coverage.get("state") not in _SL_COVERAGE: + if not isinstance(coverage.get("state"), str) or coverage.get("state") not in _SL_COVERAGE: raise SBOMLineageError("document.coverage.state is invalid") _sl_string_list(coverage.get("unknown"), "document.coverage.unknown", maximum=64) truncated = _sl_string_list(coverage.get("truncated"), "document.coverage.truncated", maximum=64) @@ -46645,7 +46705,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any def _sl_validate_node_coverage(value: Any, *, component: bool, document: bool = False, expected: Mapping[str, Any] | None = None) -> dict[str, Any]: _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "node.coverage") state = value.get("state") - if state not in _SL_COVERAGE: + if not isinstance(state, str) or state not in _SL_COVERAGE: raise SBOMLineageError("node.coverage.state is invalid") unknown = _sl_string_list(value.get("unknown"), "node.coverage.unknown", maximum=32) truncated = _sl_string_list(value.get("truncated"), "node.coverage.truncated", maximum=32) @@ -46783,7 +46843,7 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None coverage = lineage.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "truncated"}, set(), "lineage.coverage") state = coverage.get("state") - if state not in _SL_COVERAGE: + if not isinstance(state, str) or state not in _SL_COVERAGE: raise SBOMLineageError("lineage.coverage.state is invalid") unknown = _sl_string_list(coverage.get("unknown"), "lineage.coverage.unknown", maximum=32) truncated = _sl_string_list(coverage.get("truncated"), "lineage.coverage.truncated", maximum=64) @@ -47067,7 +47127,12 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query path does not terminate at its artifact") coverage = item.get("coverage") confidence = item.get("confidence") - if coverage not in _SL_COVERAGE or confidence not in _SL_CONFIDENCE: + if ( + not isinstance(coverage, str) + or not isinstance(confidence, str) + or coverage not in _SL_COVERAGE + or confidence not in _SL_CONFIDENCE + ): raise SBOMLineageError("query path result confidence or coverage is invalid") expected_path_state = _sl_path_state(checked_path) expected_path_confidence = _sl_path_confidence(checked_path) @@ -47081,7 +47146,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin coverage = query_result.get("coverage") _sl_require_keys(coverage, {"state", "path_states", "unknown", "truncated"}, set(), "query.coverage") coverage_state = coverage.get("state") - if coverage_state not in _SL_COVERAGE: + if not isinstance(coverage_state, str) or coverage_state not in _SL_COVERAGE: raise SBOMLineageError("query.coverage.state is invalid") declared_path_states = _sl_string_list(coverage.get("path_states"), "query.coverage.path_states", maximum=3) if declared_path_states != sorted(set(path_states)): diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 3e221bd7..317e1636 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -241,6 +241,16 @@ def _sl_locator_has_nonpublic_host(value: str) -> bool: return False +def _sl_query_value_has_nonpublic_host(value: str) -> bool: + candidate = value.strip() + if _sl_nonpublic_host(candidate): + return True + return any( + _sl_nonpublic_host(match.group(0)) + for match in re.finditer(r"(? bool: """Detect URI and git-style userinfo without treating version ``@`` as auth.""" for match in re.finditer(r"(?i)[a-z][a-z0-9+.-]*://([^/?#]*)", value): @@ -302,7 +312,7 @@ def _sl_private_locator(value: str) -> bool: if ( _SL_PRIVATE_LOCATOR_RE.search(query_value) or _SL_PRIVACY_MARKER_RE.search(query_value) - or _sl_locator_has_nonpublic_host(query_value) + or _sl_query_value_has_nonpublic_host(query_value) ): return True return False @@ -392,10 +402,10 @@ def _sl_safe_locator(value: Any, field: str) -> str: def _sl_safe_source_ref(value: Any, raw_bytes: bytes) -> str: - if value == "": - return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" if not isinstance(value, str): raise SBOMLineageError("source_ref must be a string") + if value == "": + return f"sha256:{hashlib.sha256(raw_bytes).hexdigest()}" text = _sl_text(value, "source_ref", required=True, limit=512) raw_digest = hashlib.sha256(raw_bytes).hexdigest() if text.startswith("sha256:source-ref:"): @@ -504,12 +514,17 @@ def _sl_xml_scalar(element: Any, names: tuple[str, ...], *, default: str = "") - if _sl_local(child).casefold() not in wanted: continue text = (child.text or "").strip() if child.text else "" - scalar = text - for key, value in child.attrib.items(): - if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value: - scalar = str(value).lstrip("#") - break - values.append(scalar) + identity_values = [ + str(value).lstrip("#") + for key, value in child.attrib.items() + if key.rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} and value + ] + if len(set(identity_values)) > 1: + raise SBOMLineageError("XML identity attributes conflict") + identity = identity_values[0] if identity_values else "" + if identity and text and identity != text: + raise SBOMLineageError("XML identity attribute conflicts with element text") + values.append(identity or text) if len(values) > 1: raise SBOMLineageError("XML singleton element is duplicated") return values[0] if values else default @@ -1121,7 +1136,12 @@ def _sl_parse_cdx_json(value: Mapping[str, Any], raw_bytes: bytes, source_ref: s creators = metadata.get("authors", []) if not isinstance(creators, list): raise SBOMLineageError("metadata.authors must be a list") - supplier = _sl_supplier(creators if creators else (metadata_component or {})) + if creators: + supplier = _sl_supplier(creators) + elif isinstance(metadata_component, Mapping): + supplier = _sl_supplier(metadata_component.get("supplier")) + else: + supplier = "" if "serialNumber" in value: document_id = _sl_id(value.get("serialNumber"), "serialNumber") else: @@ -1323,6 +1343,9 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if not match: raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") version = _sl_validate_cdx_version(match.group(1)) + expected_namespace = f"http://cyclonedx.org/schema/bom-{version}.xsd" + if namespace.casefold() != expected_namespace: + raise SBOMLineageError("CycloneDX XML namespace is not authoritative") metadata = _sl_single_child(root, "metadata") metadata_component = _sl_single_child(metadata, "component") if metadata is not None else None raw_components = [] @@ -1367,10 +1390,12 @@ def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, timestamp = _sl_xml_text(metadata, "timestamp") if metadata is not None else "" authors = _sl_single_child(metadata, "authors") if metadata is not None else None author_nodes = _sl_children(authors, "author") if authors is not None else [] - author = _sl_supplier([_sl_xml_text(item, "name") for item in author_nodes]) if author_nodes else "" + component_supplier_node = _sl_single_child(metadata_component, "supplier") if metadata_component is not None else None + component_supplier = _sl_xml_text(component_supplier_node, "name") if component_supplier_node is not None else "" + document_supplier = _sl_supplier([_sl_xml_text(item, "name") for item in author_nodes]) if author_nodes else _sl_supplier(component_supplier) return _sl_finalize_document( fmt="CycloneDX", spec_version=version, document_id=document_id, - document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=author, + document_name=_sl_xml_text(metadata, "name") if metadata is not None else "", created_at=timestamp, supplier=document_supplier, components=components, relationships=relationships, raw_bytes=raw_bytes, source_ref=source_ref, truncated=truncated, metadata_component_id=components[0]["component_id"] if metadata_component is not None else "", ) @@ -1467,46 +1492,75 @@ def _sl_json_string_size(value: str) -> int: return size -def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: - """Reject obviously oversized JSON sources before the encoder copies them.""" +def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[Any, int]: + """Snapshot one JSON-compatible value while accounting for its exact JSON size.""" active = set() if active is None else active if isinstance(value, str): - total += _sl_json_string_size(value) - elif isinstance(value, Mapping): + text = str.__str__(value) + size = _sl_json_string_size(text) + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return text, size + if isinstance(value, Mapping): marker = id(value) if marker in active: raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) - total += 2 - for index, (key, item) in enumerate(value.items()): - if not isinstance(key, str): - raise SBOMLineageError("SBOM JSON object keys must be strings") - total += (1 if index else 0) + _sl_json_string_size(key) + 1 - total = _sl_preflight_json_size(item, total=total, active=active) - if total > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") - active.remove(marker) - elif isinstance(value, (list, tuple)): + snapshot: dict[str, Any] = {} + size = 2 + try: + for index, (key, item) in enumerate(value.items()): + if not isinstance(key, str): + raise SBOMLineageError("SBOM JSON object keys must be strings") + key_text = str.__str__(key) + if key_text in snapshot: + raise SBOMLineageError("duplicate JSON object key") + child, child_size = _sl_snapshot_json(item, active=active) + size += (1 if index else 0) + _sl_json_string_size(key_text) + 1 + child_size + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + snapshot[key_text] = child + finally: + active.remove(marker) + return snapshot, size + if isinstance(value, (list, tuple)): marker = id(value) if marker in active: raise SBOMLineageError("SBOM JSON contains a cycle") active.add(marker) - total += 2 - for index, item in enumerate(value): - total += 1 if index else 0 - total = _sl_preflight_json_size(item, total=total, active=active) - if total > _SL_MAX_INPUT_BYTES: - raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") - active.remove(marker) - elif value is None: - total += 4 - elif isinstance(value, bool): - total += 4 if value else 5 - elif isinstance(value, (int, float)): + snapshot_list: list[Any] = [] + size = 2 + try: + for index, item in enumerate(value): + child, child_size = _sl_snapshot_json(item, active=active) + size += (1 if index else 0) + child_size + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + snapshot_list.append(child) + finally: + active.remove(marker) + return snapshot_list, size + if value is None: + return None, 4 + if isinstance(value, bool): + return value, 4 if value else 5 + if isinstance(value, (int, float)): + if isinstance(value, float) and not math.isfinite(value): + raise SBOMLineageError("non-finite JSON number is not allowed") try: - total += len(str(value)) + size = len(str(value)) except (ValueError, OverflowError): raise SBOMLineageError("SBOM JSON number is invalid") from None + if size > _SL_MAX_INPUT_BYTES: + raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") + return value, size + raise SBOMLineageError("SBOM JSON contains an unsupported value") + + +def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: + """Compatibility wrapper returning the exact size of a bounded JSON snapshot.""" + _, size = _sl_snapshot_json(value, active=active) + total += size if total > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") return total @@ -1629,17 +1683,18 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_bounded_byteslike(document) elif isinstance(document, Mapping): try: - _sl_preflight_json_size(document) - raw_bytes = _sl_json(document).encode("utf-8") + snapshot, _ = _sl_snapshot_json(document) + raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise except (TypeError, ValueError) as exc: raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc elif isinstance(document, str): + bounded_text = _sl_bounded_text_bytes(document) possible_path = Path(document) if "\n" not in document and possible_path.exists(): return _sl_payload(possible_path) - raw_bytes = _sl_bounded_text_bytes(document) + raw_bytes = bounded_text else: raise SBOMLineageError("SBOM input must be bytes, text, path, or object") if len(raw_bytes) > _SL_MAX_INPUT_BYTES: @@ -1857,7 +1912,12 @@ def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") raise SBOMLineageError(f"{field} direction is not allowed") if rel_type != rel_type.casefold().replace(" ", "_"): raise SBOMLineageError(f"{field}.type is not normalized") - if confidence not in _SL_CONFIDENCE or coverage not in _SL_COVERAGE: + if ( + not isinstance(confidence, str) + or not isinstance(coverage, str) + or confidence not in _SL_CONFIDENCE + or coverage not in _SL_COVERAGE + ): raise SBOMLineageError(f"{field} confidence or coverage is invalid") result: dict[str, Any] = { "from": source, "to": target, "type": rel_type, @@ -1970,7 +2030,7 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non _sl_reject_unbound_document_edges(checked_relationships, document_id) coverage = document.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "component_count", "relationship_count", "truncated", "dangling_relationships"}, set(), "document.coverage") - if coverage.get("state") not in _SL_COVERAGE: + if not isinstance(coverage.get("state"), str) or coverage.get("state") not in _SL_COVERAGE: raise SBOMLineageError("document.coverage.state is invalid") _sl_string_list(coverage.get("unknown"), "document.coverage.unknown", maximum=64) truncated = _sl_string_list(coverage.get("truncated"), "document.coverage.truncated", maximum=64) @@ -2209,7 +2269,7 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any def _sl_validate_node_coverage(value: Any, *, component: bool, document: bool = False, expected: Mapping[str, Any] | None = None) -> dict[str, Any]: _sl_require_keys(value, {"state", "unknown", "truncated"}, set(), "node.coverage") state = value.get("state") - if state not in _SL_COVERAGE: + if not isinstance(state, str) or state not in _SL_COVERAGE: raise SBOMLineageError("node.coverage.state is invalid") unknown = _sl_string_list(value.get("unknown"), "node.coverage.unknown", maximum=32) truncated = _sl_string_list(value.get("truncated"), "node.coverage.truncated", maximum=32) @@ -2347,7 +2407,7 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None coverage = lineage.get("coverage") _sl_require_keys(coverage, {"state", "unknown", "truncated"}, set(), "lineage.coverage") state = coverage.get("state") - if state not in _SL_COVERAGE: + if not isinstance(state, str) or state not in _SL_COVERAGE: raise SBOMLineageError("lineage.coverage.state is invalid") unknown = _sl_string_list(coverage.get("unknown"), "lineage.coverage.unknown", maximum=32) truncated = _sl_string_list(coverage.get("truncated"), "lineage.coverage.truncated", maximum=64) @@ -2631,7 +2691,12 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query path does not terminate at its artifact") coverage = item.get("coverage") confidence = item.get("confidence") - if coverage not in _SL_COVERAGE or confidence not in _SL_CONFIDENCE: + if ( + not isinstance(coverage, str) + or not isinstance(confidence, str) + or coverage not in _SL_COVERAGE + or confidence not in _SL_CONFIDENCE + ): raise SBOMLineageError("query path result confidence or coverage is invalid") expected_path_state = _sl_path_state(checked_path) expected_path_confidence = _sl_path_confidence(checked_path) @@ -2645,7 +2710,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin coverage = query_result.get("coverage") _sl_require_keys(coverage, {"state", "path_states", "unknown", "truncated"}, set(), "query.coverage") coverage_state = coverage.get("state") - if coverage_state not in _SL_COVERAGE: + if not isinstance(coverage_state, str) or coverage_state not in _SL_COVERAGE: raise SBOMLineageError("query.coverage.state is invalid") declared_path_states = _sl_string_list(coverage.get("path_states"), "query.coverage.path_states", maximum=3) if declared_path_states != sorted(set(path_states)): diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index eefcbb7c..190fcebb 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1690,3 +1690,128 @@ def test_oversized_lineage_receipts_fail_closed_instead_of_being_dropped(): finally: perseus._SL_LINEAGE_PROVENANCE._max_bytes = previous perseus._SL_LINEAGE_PROVENANCE.clear() + + +def test_adversarial_mapping_is_snapshotted_before_serialization(monkeypatch): + monkeypatch.setattr(perseus, "_SL_MAX_INPUT_BYTES", 64) + + class MutatingMapping(dict): + def __init__(self): + super().__init__({"bomFormat": "CycloneDX"}) + self.first = True + + def items(self): + if self.first: + self.first = False + dict.__setitem__(self, "ignored", "x" * 512) + return [("bomFormat", "CycloneDX")] + return dict.items(self) + + seen_types = [] + original_json = perseus._sl_json + + def tracked(value): + seen_types.append(type(value)) + return original_json(value) + + monkeypatch.setattr(perseus, "_sl_json", tracked) + value, raw_bytes = perseus._sl_payload(MutatingMapping()) + assert value == {"bomFormat": "CycloneDX"} + assert b"ignored" not in raw_bytes + assert MutatingMapping not in seen_types + + +def test_bare_private_ip_in_non_purl_query_is_not_persisted(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [ + {"type": "website", "url": "https://public.example/?host=10.0.0.1"}, + ] + document = perseus.ingest_sbom_document(payload) + assert "10.0.0.1" not in json.dumps(document, sort_keys=True) + + +def test_rdf_conflicting_identity_attributes_fail_closed(): + import xml.etree.ElementTree as ET + + root = ET.fromstring( + '' + '' + ) + with pytest.raises(perseus.SBOMLineageError, match="identity|conflict"): + perseus._sl_xml_scalar(root, ("endpoint",)) + + +def test_cyclonedx_supplier_uses_authors_or_component_supplier_without_raising(): + empty = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": {}, + "components": [], + } + assert perseus.ingest_sbom_document(empty)["supplier"] is None + + payload = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "metadata": { + "component": { + "bom-ref": "component:meta", + "name": "component-name", + "type": "library", + "supplier": {"name": "supplier-name"}, + }, + }, + "components": [], + } + assert perseus.ingest_sbom_document(payload)["supplier"] == "supplier-name" + + xml = ( + '' + '' + 'component-namesupplier-name' + '' + ) + assert perseus.ingest_sbom_document(xml)["supplier"] == "supplier-name" + + +def test_source_ref_rejects_non_string_custom_equality(): + class PretendsEmpty: + def __eq__(self, other): + return other == "" + + with pytest.raises(perseus.SBOMLineageError, match="source_ref"): + perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref=PretendsEmpty()) + + +def test_direct_query_rejects_unhashable_coverage_values_as_domain_error(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:unhashable-coverage") + lineage = perseus.build_sbom_lineage([document], edges=_lineage_edges()) + forged = json.loads(json.dumps(lineage)) + forged["edges"][0]["confidence"] = [] + unsigned = dict(forged) + unsigned.pop("lineage_digest", None) + forged["lineage_digest"] = perseus._sl_sha(unsigned) + with pytest.raises(perseus.SBOMLineageError, match="confidence|coverage|invalid"): + perseus.query_sbom_lineage(forged, "CVE-2021-44228", raw_documents=[raw]) + + +def test_oversized_text_is_bounded_before_path_detection(monkeypatch): + monkeypatch.setattr(perseus, "_SL_MAX_INPUT_BYTES", 16) + + class NoExistsPath(type(Path())): + def exists(self, *, follow_symlinks=True): + raise AssertionError("filesystem path detection ran before text bounds") + + monkeypatch.setattr(perseus, "Path", NoExistsPath) + with pytest.raises(perseus.SBOMLineageError, match="bytes"): + perseus._sl_payload("x" * 32) + + +def test_cyclonedx_xml_namespace_must_be_the_official_schema_namespace(): + xml = _load("cyclonedx-app.xml").decode("utf-8").replace( + "http://cyclonedx.org/schema/bom-1.5.xsd", + "https://attacker.example/schema/bom-1.5.xsd", + ) + with pytest.raises(perseus.SBOMLineageError, match="namespace"): + perseus.ingest_sbom_document(xml) From 9bfe78afce82b8ee4a40c0354ec551feeb2e1422 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 03:54:13 +0000 Subject: [PATCH 19/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 294cb50c..a5c44293 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "a2820ac-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "92889c8" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From b0300751c40c3367edf47b8bb0c92bae2cf71f05 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 04:10:37 +0000 Subject: [PATCH 20/40] fix: cover private IPs in locator paths --- perseus.py | 5 +++-- src/perseus/sbom_lineage.py | 3 ++- tests/test_sbom_lineage.py | 9 +++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/perseus.py b/perseus.py index a5c44293..6eac6508 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "92889c8" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "9bfe78a-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44683,7 +44683,7 @@ def _sl_query_value_has_nonpublic_host(value: str) -> bool: return True return any( _sl_nonpublic_host(match.group(0)) - for match in re.finditer(r"(? bool: or _SL_PRIVACY_MARKER_RE.search(decoded) or _sl_userinfo_locator(decoded) or _sl_locator_has_nonpublic_host(decoded) + or (not lowered.startswith("pkg:") and _sl_query_value_has_nonpublic_host(decoded)) ): return True if "?" in decoded: diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 317e1636..3fc78beb 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -247,7 +247,7 @@ def _sl_query_value_has_nonpublic_host(value: str) -> bool: return True return any( _sl_nonpublic_host(match.group(0)) - for match in re.finditer(r"(? bool: or _SL_PRIVACY_MARKER_RE.search(decoded) or _sl_userinfo_locator(decoded) or _sl_locator_has_nonpublic_host(decoded) + or (not lowered.startswith("pkg:") and _sl_query_value_has_nonpublic_host(decoded)) ): return True if "?" in decoded: diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 190fcebb..dd292f03 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1815,3 +1815,12 @@ def test_cyclonedx_xml_namespace_must_be_the_official_schema_namespace(): ) with pytest.raises(perseus.SBOMLineageError, match="namespace"): perseus.ingest_sbom_document(xml) + + +def test_bare_private_ip_in_non_purl_path_is_not_persisted(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [ + {"type": "website", "url": "https://public.example/path/10.0.0.1"}, + ] + document = perseus.ingest_sbom_document(payload) + assert "10.0.0.1" not in json.dumps(document, sort_keys=True) From 01f3ebdd5a18d121b5dcaca1168aa9701a18abfe Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 12:27:08 +0000 Subject: [PATCH 21/40] fix(sbom): bind raw graphs and bound XML identities --- perseus.py | 62 +++++++++++++++++++++++++++++++------ src/perseus/sbom_lineage.py | 60 +++++++++++++++++++++++++++++------ tests/test_sbom_lineage.py | 62 +++++++++++++++++++++++++++++++++++-- 3 files changed, 163 insertions(+), 21 deletions(-) diff --git a/perseus.py b/perseus.py index 6eac6508..9644f114 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "9bfe78a-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "b030075-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -45007,6 +45007,37 @@ def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) +def _sl_xml_identity(element: Any, *names: str, default: str = "") -> str: + """Resolve RDF identity while rejecting parent/child disagreement.""" + parent_values = [ + str(value).lstrip("#") + for key, value in getattr(element, "attrib", {}).items() + if ( + str(key).rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} + and value + and (str(value).startswith("#") or "://" not in str(value)) + ) + ] + if len(set(parent_values)) > 1: + raise SBOMLineageError("XML identity attributes conflict") + parent = parent_values[0] if parent_values else "" + child = _sl_xml_value(element, *names, default=default) + if parent and child and parent != child.lstrip("#"): + raise SBOMLineageError("XML parent and child identities conflict") + return child or parent or default + + +def _sl_xml_namespace(element: Any) -> str: + tag = str(getattr(element, "tag", "")) + return tag[1:].split("}", 1)[0] if tag.startswith("{") and "}" in tag else "" + + +def _sl_validate_cdx_namespace(root: Any, expected: str) -> None: + for element in root.iter(): + if _sl_xml_namespace(element) != expected: + raise SBOMLineageError("CycloneDX XML element namespace is not authoritative") + + def _sl_descendants(root: Any, *names: str) -> list[Any]: wanted = {name.casefold() for name in names} return [element for element in root.iter() if _sl_local(element).casefold() in wanted] @@ -45614,7 +45645,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, raise SBOMLineageError("multiple RDF SpdxDocument nodes are not allowed") document = documents[0] if documents else root version = _sl_validate_spdx_version(_sl_xml_value(document, "spdxVersion")) - raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") + raw_document_id = _sl_xml_identity(document, "SPDXID", "spdxid") if not raw_document_id: raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") document_id = _sl_spdx_document_id(raw_document_id) @@ -45650,7 +45681,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if ref_type.casefold() == "purl": identifiers.append(item["locator"]) truncated.extend(item for item in component_truncated if item not in truncated) - component_id = _sl_xml_value(raw, "SPDXID", "spdxid") + component_id = _sl_xml_identity(raw, "SPDXID", "spdxid") if not component_id: component_id = next((str(value).lstrip("#") for key, value in raw.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") components.append(_sl_component( @@ -45776,13 +45807,14 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: namespace = root.tag.split("}", 1)[0].lstrip("{") if "}" in root.tag else "" - match = _SL_CDX_NAMESPACE_RE.search(namespace) + match = re.fullmatch(r"http://cyclonedx\.org/schema/bom-(1\.\d+)\.xsd", namespace) if not match: raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") version = _sl_validate_cdx_version(match.group(1)) expected_namespace = f"http://cyclonedx.org/schema/bom-{version}.xsd" - if namespace.casefold() != expected_namespace: + if namespace != expected_namespace: raise SBOMLineageError("CycloneDX XML namespace is not authoritative") + _sl_validate_cdx_namespace(root, expected_namespace) metadata = _sl_single_child(root, "metadata") metadata_component = _sl_single_child(metadata, "component") if metadata is not None else None raw_components = [] @@ -45929,8 +45961,15 @@ def _sl_json_string_size(value: str) -> int: return size -def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[Any, int]: +def _sl_snapshot_json( + value: Any, + *, + active: set[int] | None = None, + depth: int = 0, +) -> tuple[Any, int]: """Snapshot one JSON-compatible value while accounting for its exact JSON size.""" + if type(depth) is not int or depth > _SL_MAX_JSON_DEPTH: + raise SBOMLineageError("SBOM JSON nesting is too deep") active = set() if active is None else active if isinstance(value, str): text = str.__str__(value) @@ -45952,7 +45991,7 @@ def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[An key_text = str.__str__(key) if key_text in snapshot: raise SBOMLineageError("duplicate JSON object key") - child, child_size = _sl_snapshot_json(item, active=active) + child, child_size = _sl_snapshot_json(item, active=active, depth=depth + 1) size += (1 if index else 0) + _sl_json_string_size(key_text) + 1 + child_size if size > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") @@ -45969,7 +46008,7 @@ def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[An size = 2 try: for index, item in enumerate(value): - child, child_size = _sl_snapshot_json(item, active=active) + child, child_size = _sl_snapshot_json(item, active=active, depth=depth + 1) size += (1 if index else 0) + child_size if size > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") @@ -46124,8 +46163,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise - except (TypeError, ValueError) as exc: - raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc + except (TypeError, ValueError, RecursionError) as exc: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from exc elif isinstance(document, str): bounded_text = _sl_bounded_text_bytes(document) possible_path = Path(document) @@ -46798,6 +46837,9 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match lineage documents within its bound") checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] + authoritative = build_sbom_lineage(checked_documents, edges=None) + if lineage.get("nodes") != authoritative.get("nodes") or lineage.get("edges") != authoritative.get("edges"): + raise SBOMLineageError("lineage graph is not bound to raw documents") document_map: dict[str, Mapping[str, Any]] = {} component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} component_ids_all: set[str] = set() diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 3fc78beb..081500f9 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -571,6 +571,37 @@ def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) +def _sl_xml_identity(element: Any, *names: str, default: str = "") -> str: + """Resolve RDF identity while rejecting parent/child disagreement.""" + parent_values = [ + str(value).lstrip("#") + for key, value in getattr(element, "attrib", {}).items() + if ( + str(key).rsplit("}", 1)[-1].casefold() in {"resource", "about", "id"} + and value + and (str(value).startswith("#") or "://" not in str(value)) + ) + ] + if len(set(parent_values)) > 1: + raise SBOMLineageError("XML identity attributes conflict") + parent = parent_values[0] if parent_values else "" + child = _sl_xml_value(element, *names, default=default) + if parent and child and parent != child.lstrip("#"): + raise SBOMLineageError("XML parent and child identities conflict") + return child or parent or default + + +def _sl_xml_namespace(element: Any) -> str: + tag = str(getattr(element, "tag", "")) + return tag[1:].split("}", 1)[0] if tag.startswith("{") and "}" in tag else "" + + +def _sl_validate_cdx_namespace(root: Any, expected: str) -> None: + for element in root.iter(): + if _sl_xml_namespace(element) != expected: + raise SBOMLineageError("CycloneDX XML element namespace is not authoritative") + + def _sl_descendants(root: Any, *names: str) -> list[Any]: wanted = {name.casefold() for name in names} return [element for element in root.iter() if _sl_local(element).casefold() in wanted] @@ -1178,7 +1209,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, raise SBOMLineageError("multiple RDF SpdxDocument nodes are not allowed") document = documents[0] if documents else root version = _sl_validate_spdx_version(_sl_xml_value(document, "spdxVersion")) - raw_document_id = _sl_xml_value(document, "SPDXID", "spdxid") + raw_document_id = _sl_xml_identity(document, "SPDXID", "spdxid") if not raw_document_id: raw_document_id = next((str(value).lstrip("#") for key, value in document.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") document_id = _sl_spdx_document_id(raw_document_id) @@ -1214,7 +1245,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, if ref_type.casefold() == "purl": identifiers.append(item["locator"]) truncated.extend(item for item in component_truncated if item not in truncated) - component_id = _sl_xml_value(raw, "SPDXID", "spdxid") + component_id = _sl_xml_identity(raw, "SPDXID", "spdxid") if not component_id: component_id = next((str(value).lstrip("#") for key, value in raw.attrib.items() if str(key).rsplit("}", 1)[-1].casefold() == "about"), "") components.append(_sl_component( @@ -1340,13 +1371,14 @@ def _sl_cdx_xml_component(raw: Any, *, truncated: list[str] | None = None, compo def _sl_parse_cdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: namespace = root.tag.split("}", 1)[0].lstrip("{") if "}" in root.tag else "" - match = _SL_CDX_NAMESPACE_RE.search(namespace) + match = re.fullmatch(r"http://cyclonedx\.org/schema/bom-(1\.\d+)\.xsd", namespace) if not match: raise SBOMLineageError("CycloneDX XML namespace must declare a supported version") version = _sl_validate_cdx_version(match.group(1)) expected_namespace = f"http://cyclonedx.org/schema/bom-{version}.xsd" - if namespace.casefold() != expected_namespace: + if namespace != expected_namespace: raise SBOMLineageError("CycloneDX XML namespace is not authoritative") + _sl_validate_cdx_namespace(root, expected_namespace) metadata = _sl_single_child(root, "metadata") metadata_component = _sl_single_child(metadata, "component") if metadata is not None else None raw_components = [] @@ -1493,8 +1525,15 @@ def _sl_json_string_size(value: str) -> int: return size -def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[Any, int]: +def _sl_snapshot_json( + value: Any, + *, + active: set[int] | None = None, + depth: int = 0, +) -> tuple[Any, int]: """Snapshot one JSON-compatible value while accounting for its exact JSON size.""" + if type(depth) is not int or depth > _SL_MAX_JSON_DEPTH: + raise SBOMLineageError("SBOM JSON nesting is too deep") active = set() if active is None else active if isinstance(value, str): text = str.__str__(value) @@ -1516,7 +1555,7 @@ def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[An key_text = str.__str__(key) if key_text in snapshot: raise SBOMLineageError("duplicate JSON object key") - child, child_size = _sl_snapshot_json(item, active=active) + child, child_size = _sl_snapshot_json(item, active=active, depth=depth + 1) size += (1 if index else 0) + _sl_json_string_size(key_text) + 1 + child_size if size > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") @@ -1533,7 +1572,7 @@ def _sl_snapshot_json(value: Any, *, active: set[int] | None = None) -> tuple[An size = 2 try: for index, item in enumerate(value): - child, child_size = _sl_snapshot_json(item, active=active) + child, child_size = _sl_snapshot_json(item, active=active, depth=depth + 1) size += (1 if index else 0) + child_size if size > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") @@ -1688,8 +1727,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise - except (TypeError, ValueError) as exc: - raise SBOMLineageError("SBOM mapping is not canonical JSON") from exc + except (TypeError, ValueError, RecursionError) as exc: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from exc elif isinstance(document, str): bounded_text = _sl_bounded_text_bytes(document) possible_path = Path(document) @@ -2362,6 +2401,9 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match lineage documents within its bound") checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] + authoritative = build_sbom_lineage(checked_documents, edges=None) + if lineage.get("nodes") != authoritative.get("nodes") or lineage.get("edges") != authoritative.get("edges"): + raise SBOMLineageError("lineage graph is not bound to raw documents") document_map: dict[str, Mapping[str, Any]] = {} component_map: dict[str, list[tuple[str, Mapping[str, Any]]]] = {} component_ids_all: set[str] = set() diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index dd292f03..d7c367b5 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -704,6 +704,34 @@ def test_rebinding_accepts_an_already_sanitized_source_reference(): assert perseus.verify_sbom_lineage(lineage)["valid"] is True +def test_raw_documents_reject_recommitted_forged_graph_edges(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:raw-graph") + lineage = perseus.build_sbom_lineage([document], edges=[], raw_documents=[raw]) + forged = json.loads(json.dumps(lineage)) + component_id = document["components"][0]["component_id"] + forged["nodes"].append({ + "node_id": "artifact:forged", + "kind": "artifact", + "coverage": {"state": "partial", "unknown": ["node_metadata"], "truncated": []}, + }) + forged["edges"].append({ + "from": component_id, + "to": "artifact:forged", + "type": "generates", + "confidence": "high", + "coverage": "complete", + "evidence_refs": ["ledger:forged"], + }) + forged["coverage"] = {"state": "partial", "unknown": ["external_lineage"], "truncated": []} + unsigned = dict(forged) + unsigned.pop("lineage_digest", None) + forged["lineage_digest"] = perseus._sl_sha(unsigned) + result = perseus.verify_sbom_lineage(forged, raw_documents=[raw]) + assert result["valid"] is False + assert "graph" in result["error"] or "edge" in result["error"] + + def test_cli_merge_rebinds_normalized_documents_with_raw_documents(tmp_path, capsys): raw = _load("spdx-app.json") raw_path = tmp_path / "raw.json" @@ -1721,6 +1749,14 @@ def tracked(value): assert MutatingMapping not in seen_types +def test_deep_mapping_is_rejected_as_bounded_sbom_error(): + value = {"leaf": 0} + for _ in range(2000): + value = {"nested": value} + with pytest.raises(perseus.SBOMLineageError, match="nesting|depth"): + perseus._sl_payload(value) + + def test_bare_private_ip_in_non_purl_query_is_not_persisted(): payload = json.loads(_load("cyclonedx-app.json")) payload["components"][0]["externalReferences"] = [ @@ -1741,7 +1777,29 @@ def test_rdf_conflicting_identity_attributes_fail_closed(): perseus._sl_xml_scalar(root, ("endpoint",)) -def test_cyclonedx_supplier_uses_authors_or_component_supplier_without_raising(): +def test_rdf_parent_about_conflicting_child_spdxid_fails_closed(): + import xml.etree.ElementTree as ET + + root = ET.fromstring( + '' + 'SPDX-2.3SPDXRef-DOCUMENT-B' + '2024-01-01T00:00:00Z' + ) + with pytest.raises(perseus.SBOMLineageError, match="identity|conflict"): + perseus._sl_spdx_rdf_xml(root, b"", "artifact:rdf-conflict") + + +def test_cyclonedx_xml_rejects_attacker_namespace_child(): + xml = ( + '' + 'evil' + '' + ) + with pytest.raises(perseus.SBOMLineageError, match="namespace"): + perseus.ingest_sbom_document(xml) + + empty = { "bomFormat": "CycloneDX", "specVersion": "1.5", @@ -1792,7 +1850,7 @@ def test_direct_query_rejects_unhashable_coverage_values_as_domain_error(): unsigned = dict(forged) unsigned.pop("lineage_digest", None) forged["lineage_digest"] = perseus._sl_sha(unsigned) - with pytest.raises(perseus.SBOMLineageError, match="confidence|coverage|invalid"): + with pytest.raises(perseus.SBOMLineageError, match="confidence|coverage|invalid|graph|edge"): perseus.query_sbom_lineage(forged, "CVE-2021-44228", raw_documents=[raw]) From 60aaa1c8a5230c37205e37a6cb6954c128c60c38 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 12:27:30 +0000 Subject: [PATCH 22/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 9644f114..d7b5bdbb 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "b030075-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "01f3ebd" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 98b164cc8ae5e517f7a8d8fe4a8d9aa6e69d1e0e Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 12:49:36 +0000 Subject: [PATCH 23/40] fix(sbom): bind external edges and IPv6 privacy --- perseus.py | 86 ++++++++++++++++++++++++++++--------- src/perseus/cli.py | 1 + src/perseus/sbom_lineage.py | 83 ++++++++++++++++++++++++++--------- tests/test_sbom_lineage.py | 49 +++++++++++++++++++++ 4 files changed, 178 insertions(+), 41 deletions(-) diff --git a/perseus.py b/perseus.py index d7b5bdbb..c8e293b3 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "01f3ebd" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "60aaa1c-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44681,10 +44681,13 @@ def _sl_query_value_has_nonpublic_host(value: str) -> bool: candidate = value.strip() if _sl_nonpublic_host(candidate): return True - return any( - _sl_nonpublic_host(match.group(0)) - for match in re.finditer(r"(? bool: @@ -46163,7 +46166,7 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise - except (TypeError, ValueError, RecursionError) as exc: + except Exception as exc: raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from exc elif isinstance(document, str): bounded_text = _sl_bounded_text_bytes(document) @@ -46812,7 +46815,13 @@ def _sl_validate_lineage_node( return node_id -def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None = None) -> dict[str, Any]: +def _sl_loaded_lineage( + lineage: Mapping[str, Any], + *, + raw_documents: Any | None = None, + edges: Any | None = None, +) -> dict[str, Any]: + edge_input = edges required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: raise SBOMLineageError("unsupported software-lineage schema") @@ -46837,7 +46846,7 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match lineage documents within its bound") checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] - authoritative = build_sbom_lineage(checked_documents, edges=None) + authoritative = build_sbom_lineage(checked_documents, edges=edge_input) if lineage.get("nodes") != authoritative.get("nodes") or lineage.get("edges") != authoritative.get("edges"): raise SBOMLineageError("lineage graph is not bound to raw documents") document_map: dict[str, Mapping[str, Any]] = {} @@ -46903,9 +46912,14 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None return dict(lineage) -def verify_sbom_lineage(lineage: Mapping[str, Any], raw_documents: Any | None = None) -> dict[str, Any]: +def verify_sbom_lineage( + lineage: Mapping[str, Any], + raw_documents: Any | None = None, + *, + edges: Any | None = None, +) -> dict[str, Any]: try: - loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents, edges=edges) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -46946,9 +46960,16 @@ def _sl_path_confidence(path: list[Mapping[str, Any]]) -> str: return "high" -def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32, raw_documents: Any | None = None) -> dict[str, Any]: +def query_sbom_lineage( + lineage: Mapping[str, Any], + query: str, + *, + limit: int = 32, + raw_documents: Any | None = None, + edges: Any | None = None, +) -> dict[str, Any]: """Find impacted artifact nodes and return every traversed evidence edge.""" - loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents, edges=edges) limit = _sl_limit(limit, "limit") query_text = _sl_strict_text(query, "query", limit=512) assert query_text is not None @@ -47049,7 +47070,12 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 return unsigned -def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: +def _sl_validate_query_result( + query_result: Mapping[str, Any], + authoritative_lineage: Mapping[str, Any] | None = None, + raw_documents: Any | None = None, + edges: Any | None = None, +) -> dict[str, Any]: required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") @@ -47067,7 +47093,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query authoritative lineage exceeds its bound") authority = _SL_LINEAGE_PROVENANCE.get(lineage_digest) if authoritative_lineage is not None: - loaded_authority = _sl_loaded_lineage(authoritative_lineage, raw_documents=raw_documents) + loaded_authority = _sl_loaded_lineage(authoritative_lineage, raw_documents=raw_documents, edges=edges) if lineage_digest != loaded_authority["lineage_digest"]: raise SBOMLineageError("query result lineage digest does not match the authoritative lineage") authority = ( @@ -47218,7 +47244,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query status is inconsistent with coverage") assert authoritative_lineage is not None expected_result = query_sbom_lineage( - authoritative_lineage, query, limit=limit, raw_documents=raw_documents, + authoritative_lineage, query, limit=limit, raw_documents=raw_documents, edges=edges, ) for field in ( "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", @@ -47229,9 +47255,15 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin return dict(query_result) -def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: +def verify_sbom_lineage_query( + query_result: Mapping[str, Any], + authoritative_lineage: Mapping[str, Any] | None = None, + raw_documents: Any | None = None, + *, + edges: Any | None = None, +) -> dict[str, Any]: try: - checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents) + checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents, edges) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} @@ -47326,11 +47358,16 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: else: edges = [] document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) - _sl_loaded_lineage(document, raw_documents=raw_inputs) + _sl_loaded_lineage(document, raw_documents=raw_inputs, edges=edges) elif command == "query": raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage_path = _sl_cli_checked_path(getattr(args, "lineage", None), "lineage") - _sl_cli_preflight_paths((("lineage", [lineage_path]), ("raw_documents", raw_document_paths))) + edge_path = _sl_cli_checked_path(getattr(args, "edges", None), "edges") if getattr(args, "edges", None) else None + _sl_cli_preflight_paths(( + ("lineage", [lineage_path]), + ("raw_documents", raw_document_paths), + ("edges", [edge_path] if edge_path is not None else None), + )) actual_input_bytes = 0 lineage, lineage_bytes = _sl_load_json_bounded(Path(lineage_path)) actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, lineage_bytes) @@ -47346,10 +47383,16 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: _, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) raw_inputs.append(raw_bytes) + edges = None + if edge_path is not None: + edges, edge_bytes = _sl_load_json_bounded(Path(edge_path)) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, edge_bytes) + if not isinstance(edges, list): + raise SBOMLineageError("lineage edges JSON must be a list") document = query_sbom_lineage( - lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, + lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, edges=edges, ) - _sl_validate_query_result(document, lineage, raw_inputs) + _sl_validate_query_result(document, lineage, raw_inputs, edges) else: raise SBOMLineageError("command must be ingest, merge, or query") serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" @@ -48246,6 +48289,7 @@ def main(): p_sbom_query.add_argument("component", help="Component name, version, purl, or vulnerability reference") p_sbom_query.add_argument("--limit", type=int, default=32, help="Maximum impacted artifacts") p_sbom_query.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to lineage documents; required across processes") + p_sbom_query.add_argument("--edges", default=None, help="Optional JSON file containing the authoritative external edges used to build the lineage") p_sbom_query.add_argument("--output", "-o", default=None, help="Write the query result to a JSON file") p_sbom_query.add_argument("--json", action="store_true", help="Print machine-readable JSON") diff --git a/src/perseus/cli.py b/src/perseus/cli.py index fbf726c3..109478c0 100644 --- a/src/perseus/cli.py +++ b/src/perseus/cli.py @@ -171,6 +171,7 @@ def main(): p_sbom_query.add_argument("component", help="Component name, version, purl, or vulnerability reference") p_sbom_query.add_argument("--limit", type=int, default=32, help="Maximum impacted artifacts") p_sbom_query.add_argument("--raw-documents", nargs="+", default=None, dest="raw_documents", help="Raw source paths corresponding to lineage documents; required across processes") + p_sbom_query.add_argument("--edges", default=None, help="Optional JSON file containing the authoritative external edges used to build the lineage") p_sbom_query.add_argument("--output", "-o", default=None, help="Write the query result to a JSON file") p_sbom_query.add_argument("--json", action="store_true", help="Print machine-readable JSON") diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 081500f9..8eb3fe60 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -245,10 +245,13 @@ def _sl_query_value_has_nonpublic_host(value: str) -> bool: candidate = value.strip() if _sl_nonpublic_host(candidate): return True - return any( - _sl_nonpublic_host(match.group(0)) - for match in re.finditer(r"(? bool: @@ -1727,7 +1730,7 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise - except (TypeError, ValueError, RecursionError) as exc: + except Exception as exc: raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from exc elif isinstance(document, str): bounded_text = _sl_bounded_text_bytes(document) @@ -2376,7 +2379,13 @@ def _sl_validate_lineage_node( return node_id -def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None = None) -> dict[str, Any]: +def _sl_loaded_lineage( + lineage: Mapping[str, Any], + *, + raw_documents: Any | None = None, + edges: Any | None = None, +) -> dict[str, Any]: + edge_input = edges required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: raise SBOMLineageError("unsupported software-lineage schema") @@ -2401,7 +2410,7 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None if not isinstance(raw_documents, (list, tuple)) or len(raw_documents) != len(documents) or len(raw_documents) > _SL_MAX_DOCUMENTS: raise SBOMLineageError("raw_documents must match lineage documents within its bound") checked_documents = [_sl_rebind_document(document, raw) for document, raw in zip(documents, raw_documents)] - authoritative = build_sbom_lineage(checked_documents, edges=None) + authoritative = build_sbom_lineage(checked_documents, edges=edge_input) if lineage.get("nodes") != authoritative.get("nodes") or lineage.get("edges") != authoritative.get("edges"): raise SBOMLineageError("lineage graph is not bound to raw documents") document_map: dict[str, Mapping[str, Any]] = {} @@ -2467,9 +2476,14 @@ def _sl_loaded_lineage(lineage: Mapping[str, Any], *, raw_documents: Any | None return dict(lineage) -def verify_sbom_lineage(lineage: Mapping[str, Any], raw_documents: Any | None = None) -> dict[str, Any]: +def verify_sbom_lineage( + lineage: Mapping[str, Any], + raw_documents: Any | None = None, + *, + edges: Any | None = None, +) -> dict[str, Any]: try: - loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents, edges=edges) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -2510,9 +2524,16 @@ def _sl_path_confidence(path: list[Mapping[str, Any]]) -> str: return "high" -def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 32, raw_documents: Any | None = None) -> dict[str, Any]: +def query_sbom_lineage( + lineage: Mapping[str, Any], + query: str, + *, + limit: int = 32, + raw_documents: Any | None = None, + edges: Any | None = None, +) -> dict[str, Any]: """Find impacted artifact nodes and return every traversed evidence edge.""" - loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents) + loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents, edges=edges) limit = _sl_limit(limit, "limit") query_text = _sl_strict_text(query, "query", limit=512) assert query_text is not None @@ -2613,7 +2634,12 @@ def query_sbom_lineage(lineage: Mapping[str, Any], query: str, *, limit: int = 3 return unsigned -def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: +def _sl_validate_query_result( + query_result: Mapping[str, Any], + authoritative_lineage: Mapping[str, Any] | None = None, + raw_documents: Any | None = None, + edges: Any | None = None, +) -> dict[str, Any]: required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") @@ -2631,7 +2657,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query authoritative lineage exceeds its bound") authority = _SL_LINEAGE_PROVENANCE.get(lineage_digest) if authoritative_lineage is not None: - loaded_authority = _sl_loaded_lineage(authoritative_lineage, raw_documents=raw_documents) + loaded_authority = _sl_loaded_lineage(authoritative_lineage, raw_documents=raw_documents, edges=edges) if lineage_digest != loaded_authority["lineage_digest"]: raise SBOMLineageError("query result lineage digest does not match the authoritative lineage") authority = ( @@ -2782,7 +2808,7 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin raise SBOMLineageError("query status is inconsistent with coverage") assert authoritative_lineage is not None expected_result = query_sbom_lineage( - authoritative_lineage, query, limit=limit, raw_documents=raw_documents, + authoritative_lineage, query, limit=limit, raw_documents=raw_documents, edges=edges, ) for field in ( "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", @@ -2793,9 +2819,15 @@ def _sl_validate_query_result(query_result: Mapping[str, Any], authoritative_lin return dict(query_result) -def verify_sbom_lineage_query(query_result: Mapping[str, Any], authoritative_lineage: Mapping[str, Any] | None = None, raw_documents: Any | None = None) -> dict[str, Any]: +def verify_sbom_lineage_query( + query_result: Mapping[str, Any], + authoritative_lineage: Mapping[str, Any] | None = None, + raw_documents: Any | None = None, + *, + edges: Any | None = None, +) -> dict[str, Any]: try: - checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents) + checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents, edges) except (SBOMLineageError, TypeError, ValueError) as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} @@ -2890,11 +2922,16 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: else: edges = [] document = build_sbom_lineage(documents, edges=edges, raw_documents=raw_inputs) - _sl_loaded_lineage(document, raw_documents=raw_inputs) + _sl_loaded_lineage(document, raw_documents=raw_inputs, edges=edges) elif command == "query": raw_document_paths = _sl_cli_bounded_paths(getattr(args, "raw_documents", None), "raw_documents") lineage_path = _sl_cli_checked_path(getattr(args, "lineage", None), "lineage") - _sl_cli_preflight_paths((("lineage", [lineage_path]), ("raw_documents", raw_document_paths))) + edge_path = _sl_cli_checked_path(getattr(args, "edges", None), "edges") if getattr(args, "edges", None) else None + _sl_cli_preflight_paths(( + ("lineage", [lineage_path]), + ("raw_documents", raw_document_paths), + ("edges", [edge_path] if edge_path is not None else None), + )) actual_input_bytes = 0 lineage, lineage_bytes = _sl_load_json_bounded(Path(lineage_path)) actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, lineage_bytes) @@ -2910,10 +2947,16 @@ def cmd_sbom(args: Any, cfg: Mapping[str, Any] | None = None) -> int: _, raw_bytes = _sl_load_json_bounded(Path(path), allow_non_json=True) actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, raw_bytes) raw_inputs.append(raw_bytes) + edges = None + if edge_path is not None: + edges, edge_bytes = _sl_load_json_bounded(Path(edge_path)) + actual_input_bytes = _sl_account_input_bytes(actual_input_bytes, edge_bytes) + if not isinstance(edges, list): + raise SBOMLineageError("lineage edges JSON must be a list") document = query_sbom_lineage( - lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, + lineage, args.component, limit=getattr(args, "limit", 32), raw_documents=raw_inputs, edges=edges, ) - _sl_validate_query_result(document, lineage, raw_inputs) + _sl_validate_query_result(document, lineage, raw_inputs, edges) else: raise SBOMLineageError("command must be ingest, merge, or query") serialized = json.dumps(document, indent=2, sort_keys=True) + "\n" diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index d7c367b5..423f2571 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -732,6 +732,23 @@ def test_raw_documents_reject_recommitted_forged_graph_edges(): assert "graph" in result["error"] or "edge" in result["error"] +def test_raw_rebinding_accepts_authoritative_external_edges_when_supplied(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:raw-edge-receipt") + component_id = document["components"][0]["component_id"] + edges = [{ + "from": component_id, + "to": "artifact:external-receipt", + "type": "generates", + "confidence": "high", + "coverage": "complete", + "evidence_refs": ["ledger:external-receipt"], + }] + lineage = perseus.build_sbom_lineage([document], edges=edges, raw_documents=[raw]) + assert perseus.verify_sbom_lineage(lineage, raw_documents=[raw])["valid"] is False + assert perseus.verify_sbom_lineage(lineage, raw_documents=[raw], edges=edges)["valid"] is True + + def test_cli_merge_rebinds_normalized_documents_with_raw_documents(tmp_path, capsys): raw = _load("spdx-app.json") raw_path = tmp_path / "raw.json" @@ -1757,6 +1774,26 @@ def test_deep_mapping_is_rejected_as_bounded_sbom_error(): perseus._sl_payload(value) +def test_hostile_mapping_exceptions_are_bounded_domain_errors(): + from collections.abc import Mapping + + class ExplodingMapping(Mapping): + def __getitem__(self, key): + raise RuntimeError("RAW-MAPPING-SENTINEL") + + def __iter__(self): + return iter(("bomFormat",)) + + def __len__(self): + return 1 + + def items(self): + raise RuntimeError("RAW-MAPPING-SENTINEL") + + with pytest.raises(perseus.SBOMLineageError, match="mapping|canonical|bounded"): + perseus._sl_payload(ExplodingMapping()) + + def test_bare_private_ip_in_non_purl_query_is_not_persisted(): payload = json.loads(_load("cyclonedx-app.json")) payload["components"][0]["externalReferences"] = [ @@ -1882,3 +1919,15 @@ def test_bare_private_ip_in_non_purl_path_is_not_persisted(): ] document = perseus.ingest_sbom_document(payload) assert "10.0.0.1" not in json.dumps(document, sort_keys=True) + + +def test_bare_private_ipv6_literals_in_non_purl_paths_are_not_persisted(): + payload = json.loads(_load("cyclonedx-app.json")) + payload["components"][0]["externalReferences"] = [ + {"type": "website", "url": "https://public.example/path/::1"}, + {"type": "website", "url": "https://public.example/path/%3A%3A1"}, + ] + document = perseus.ingest_sbom_document(payload) + serialized = json.dumps(document, sort_keys=True) + assert "::1" not in serialized + assert "%3A%3A1" not in serialized From 686c14ac87d978b32022329aa77c713884ae0319 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 12:49:50 +0000 Subject: [PATCH 24/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index c8e293b3..9eb6026b 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "60aaa1c-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "98b164c" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From d49a6382b17339b692bc7e06a236696156947881 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 13:08:15 +0000 Subject: [PATCH 25/40] fix(sbom): bound hostile mappings and document edge receipts --- docs/SBOM.md | 2 +- perseus.py | 32 +++++++++++++++--- src/perseus/sbom_lineage.py | 30 ++++++++++++++--- tests/test_sbom_lineage.py | 66 ++++++++++++++++++++++++++++++++++--- 4 files changed, 116 insertions(+), 14 deletions(-) diff --git a/docs/SBOM.md b/docs/SBOM.md index 54458d8b..67be7a3e 100644 --- a/docs/SBOM.md +++ b/docs/SBOM.md @@ -135,7 +135,7 @@ perseus sbom ingest build.spdx.json --output normalized.json # Persisted normalized documents must be rebound to their raw source at merge. perseus sbom merge normalized.json --raw-documents build.spdx.json --edges pipeline-edges.json --output lineage.json # Persisted lineage must be rebound again at query in a new process. -perseus sbom query lineage.json CVE-2021-44228 --raw-documents build.spdx.json --json +perseus sbom query lineage.json CVE-2021-44228 --raw-documents build.spdx.json --edges pipeline-edges.json --json ``` The core path requires no cloud service. Deterministic JSON/XML fixtures and diff --git a/perseus.py b/perseus.py index 9eb6026b..6a2b9d12 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "98b164c" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "686c14a-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -46036,6 +46036,20 @@ def _sl_snapshot_json( raise SBOMLineageError("SBOM JSON contains an unsupported value") +def _sl_snapshot_mapping(value: Any, field: str) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise SBOMLineageError(f"{field} must be an object") + try: + snapshot, _ = _sl_snapshot_json(value) + except SBOMLineageError: + raise + except Exception: + raise SBOMLineageError(f"{field} is not bounded canonical JSON") from None + if not isinstance(snapshot, dict): + raise SBOMLineageError(f"{field} must be an object") + return snapshot + + def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: """Compatibility wrapper returning the exact size of a bounded JSON snapshot.""" _, size = _sl_snapshot_json(value, active=active) @@ -46374,6 +46388,8 @@ def _sl_validate_component(component: Any) -> dict[str, Any]: def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") -> dict[str, Any]: + if isinstance(relationship, Mapping): + relationship = _sl_snapshot_mapping(relationship, field) _sl_require_keys(relationship, {"from", "to", "type", "confidence", "coverage"}, {"evidence_refs"}, field) source = _sl_strict_text(relationship.get("from"), f"{field}.from", limit=256) target = _sl_strict_text(relationship.get("to"), f"{field}.to", limit=256) @@ -46448,6 +46464,8 @@ def _sl_expected_document_coverage(document_id: str, fmt: str, document_name: st def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | None = None) -> dict[str, Any]: + if isinstance(document, Mapping): + document = _sl_snapshot_mapping(document, "normalized SBOM document") required = { "schema_version", "format", "spec_version", "document_id", "document_name", "document_sha256", "source_ref", "created_at", "supplier", "components", @@ -46538,7 +46556,7 @@ def _sl_rebind_document(document: Mapping[str, Any], raw_document: Any) -> dict[ def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = None) -> dict[str, Any]: try: checked = _sl_validate_document(document) if raw_document is None else _sl_rebind_document(document, raw_document) - except (SBOMLineageError, TypeError, ValueError) as exc: + except Exception as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} @@ -46567,7 +46585,7 @@ def _sl_bounded_external_edges(edges: Any) -> list[Any]: raise SBOMLineageError("lineage edges exceed their bound") if not isinstance(raw, Mapping): raise SBOMLineageError("lineage edges must contain objects") - result.append(raw) + result.append(_sl_snapshot_mapping(raw, "lineage edge")) except SBOMLineageError: raise except Exception: @@ -46649,6 +46667,8 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any truncated.append("documents") normalized = [] for index, document in enumerate(documents[:_SL_MAX_DOCUMENTS]): + if isinstance(document, Mapping): + document = _sl_snapshot_mapping(document, "SBOM document") if isinstance(document, Mapping) and document.get("schema_version") == _SL_SCHEMA: if rebound_raw is None: normalized.append(_sl_validate_document(document)) @@ -46821,6 +46841,7 @@ def _sl_loaded_lineage( raw_documents: Any | None = None, edges: Any | None = None, ) -> dict[str, Any]: + lineage = _sl_snapshot_mapping(lineage, "lineage") edge_input = edges required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: @@ -46920,7 +46941,7 @@ def verify_sbom_lineage( ) -> dict[str, Any]: try: loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents, edges=edges) - except (SBOMLineageError, TypeError, ValueError) as exc: + except Exception as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -47076,6 +47097,7 @@ def _sl_validate_query_result( raw_documents: Any | None = None, edges: Any | None = None, ) -> dict[str, Any]: + query_result = _sl_snapshot_mapping(query_result, "query result") required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") @@ -47264,7 +47286,7 @@ def verify_sbom_lineage_query( ) -> dict[str, Any]: try: checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents, edges) - except (SBOMLineageError, TypeError, ValueError) as exc: + except Exception as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 8eb3fe60..5e3cae2f 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -1600,6 +1600,20 @@ def _sl_snapshot_json( raise SBOMLineageError("SBOM JSON contains an unsupported value") +def _sl_snapshot_mapping(value: Any, field: str) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise SBOMLineageError(f"{field} must be an object") + try: + snapshot, _ = _sl_snapshot_json(value) + except SBOMLineageError: + raise + except Exception: + raise SBOMLineageError(f"{field} is not bounded canonical JSON") from None + if not isinstance(snapshot, dict): + raise SBOMLineageError(f"{field} must be an object") + return snapshot + + def _sl_preflight_json_size(value: Any, *, total: int = 0, active: set[int] | None = None) -> int: """Compatibility wrapper returning the exact size of a bounded JSON snapshot.""" _, size = _sl_snapshot_json(value, active=active) @@ -1938,6 +1952,8 @@ def _sl_validate_component(component: Any) -> dict[str, Any]: def _sl_validate_relationship(relationship: Any, *, field: str = "relationship") -> dict[str, Any]: + if isinstance(relationship, Mapping): + relationship = _sl_snapshot_mapping(relationship, field) _sl_require_keys(relationship, {"from", "to", "type", "confidence", "coverage"}, {"evidence_refs"}, field) source = _sl_strict_text(relationship.get("from"), f"{field}.from", limit=256) target = _sl_strict_text(relationship.get("to"), f"{field}.to", limit=256) @@ -2012,6 +2028,8 @@ def _sl_expected_document_coverage(document_id: str, fmt: str, document_name: st def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | None = None) -> dict[str, Any]: + if isinstance(document, Mapping): + document = _sl_snapshot_mapping(document, "normalized SBOM document") required = { "schema_version", "format", "spec_version", "document_id", "document_name", "document_sha256", "source_ref", "created_at", "supplier", "components", @@ -2102,7 +2120,7 @@ def _sl_rebind_document(document: Mapping[str, Any], raw_document: Any) -> dict[ def verify_sbom_document(document: Mapping[str, Any], raw_document: Any | None = None) -> dict[str, Any]: try: checked = _sl_validate_document(document) if raw_document is None else _sl_rebind_document(document, raw_document) - except (SBOMLineageError, TypeError, ValueError) as exc: + except Exception as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "ingestion_digest": checked["ingestion_digest"], "component_count": len(checked["components"]), "relationship_count": len(checked["relationships"])} @@ -2131,7 +2149,7 @@ def _sl_bounded_external_edges(edges: Any) -> list[Any]: raise SBOMLineageError("lineage edges exceed their bound") if not isinstance(raw, Mapping): raise SBOMLineageError("lineage edges must contain objects") - result.append(raw) + result.append(_sl_snapshot_mapping(raw, "lineage edge")) except SBOMLineageError: raise except Exception: @@ -2213,6 +2231,8 @@ def build_sbom_lineage(documents: Any, *, edges: Any = None, raw_documents: Any truncated.append("documents") normalized = [] for index, document in enumerate(documents[:_SL_MAX_DOCUMENTS]): + if isinstance(document, Mapping): + document = _sl_snapshot_mapping(document, "SBOM document") if isinstance(document, Mapping) and document.get("schema_version") == _SL_SCHEMA: if rebound_raw is None: normalized.append(_sl_validate_document(document)) @@ -2385,6 +2405,7 @@ def _sl_loaded_lineage( raw_documents: Any | None = None, edges: Any | None = None, ) -> dict[str, Any]: + lineage = _sl_snapshot_mapping(lineage, "lineage") edge_input = edges required = {"schema_version", "documents", "nodes", "edges", "coverage", "lineage_digest"} if not isinstance(lineage, Mapping) or set(lineage) != required or lineage.get("schema_version") != _SL_LINEAGE_SCHEMA: @@ -2484,7 +2505,7 @@ def verify_sbom_lineage( ) -> dict[str, Any]: try: loaded = _sl_loaded_lineage(lineage, raw_documents=raw_documents, edges=edges) - except (SBOMLineageError, TypeError, ValueError) as exc: + except Exception as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": loaded["schema_version"], "lineage_digest": loaded["lineage_digest"], "node_count": len(loaded.get("nodes", [])), "edge_count": len(loaded.get("edges", []))} @@ -2640,6 +2661,7 @@ def _sl_validate_query_result( raw_documents: Any | None = None, edges: Any | None = None, ) -> dict[str, Any]: + query_result = _sl_snapshot_mapping(query_result, "query result") required = {"schema_version", "lineage_digest", "lineage_nodes", "lineage_edges", "query", "limit", "matched_nodes", "impacted_artifacts", "status", "coverage", "claims", "query_digest"} if not isinstance(query_result, Mapping) or set(query_result) != required or query_result.get("schema_version") != _SL_QUERY_SCHEMA: raise SBOMLineageError("unsupported query schema") @@ -2828,7 +2850,7 @@ def verify_sbom_lineage_query( ) -> dict[str, Any]: try: checked = _sl_validate_query_result(query_result, authoritative_lineage, raw_documents, edges) - except (SBOMLineageError, TypeError, ValueError) as exc: + except Exception as exc: return {"valid": False, "error": _sl_public_error(exc)} return {"valid": True, "schema_version": checked["schema_version"], "query_digest": checked["query_digest"], "expected_digest": checked["query_digest"]} diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 423f2571..e55963f7 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -970,7 +970,16 @@ def test_documented_cli_workflow_rebinds_raw_sources_across_processes(tmp_path): normalized_path = tmp_path / "normalized.json" lineage_path = tmp_path / "lineage.json" query_path = tmp_path / "query.json" + edges_path = tmp_path / "pipeline-edges.json" raw_path.write_bytes(_load("spdx-app.json")) + edges_path.write_text(json.dumps([{ + "from": "SPDXRef-Log4j", + "to": "artifact:pipeline-image", + "type": "generates", + "confidence": "high", + "coverage": "complete", + "evidence_refs": ["ledger:pipeline-001"], + }]), encoding="utf-8") cli = [sys.executable, str(ROOT / "perseus.py"), "sbom"] ingest = subprocess.run( @@ -980,7 +989,7 @@ def test_documented_cli_workflow_rebinds_raw_sources_across_processes(tmp_path): assert ingest.returncode == 0, ingest.stdout + ingest.stderr merge = subprocess.run( - cli + ["merge", str(normalized_path), "--raw-documents", str(raw_path), "--output", str(lineage_path), "--json"], + cli + ["merge", str(normalized_path), "--raw-documents", str(raw_path), "--edges", str(edges_path), "--output", str(lineage_path), "--json"], capture_output=True, text=True, check=False, ) assert merge.returncode == 0, merge.stdout + merge.stderr @@ -993,16 +1002,17 @@ def test_documented_cli_workflow_rebinds_raw_sources_across_processes(tmp_path): assert json.loads(without_raw.stdout)["valid"] is False query = subprocess.run( - cli + ["query", str(lineage_path), "CVE-2021-44228", "--raw-documents", str(raw_path), "--output", str(query_path), "--json"], + cli + ["query", str(lineage_path), "CVE-2021-44228", "--raw-documents", str(raw_path), "--edges", str(edges_path), "--output", str(query_path), "--json"], capture_output=True, text=True, check=False, ) assert query.returncode == 0, query.stdout + query.stderr assert json.loads(query_path.read_text(encoding="utf-8"))["query"] == "CVE-2021-44228" + assert any(item["artifact_id"] == "artifact:pipeline-image" for item in json.loads(query_path.read_text(encoding="utf-8"))["impacted_artifacts"]) verify = subprocess.run([ sys.executable, "-c", - "import json,sys,perseus; raw=open(sys.argv[3],'rb').read(); lineage=json.load(open(sys.argv[1])); query=json.load(open(sys.argv[2])); print(json.dumps([perseus.verify_sbom_lineage(lineage, raw_documents=[raw]), perseus.verify_sbom_lineage_query(query, lineage, [raw])]))", - str(lineage_path), str(query_path), str(raw_path), + "import json,sys,perseus; raw=open(sys.argv[3],'rb').read(); lineage=json.load(open(sys.argv[1])); query=json.load(open(sys.argv[2])); edges=json.load(open(sys.argv[4])); print(json.dumps([perseus.verify_sbom_lineage(lineage, raw_documents=[raw], edges=edges), perseus.verify_sbom_lineage_query(query, lineage, [raw], edges=edges)]))", + str(lineage_path), str(query_path), str(raw_path), str(edges_path), ], capture_output=True, text=True, check=False, cwd=str(ROOT)) assert verify.returncode == 0, verify.stdout + verify.stderr checks = json.loads(verify.stdout) @@ -1794,6 +1804,54 @@ def items(self): perseus._sl_payload(ExplodingMapping()) +@pytest.mark.parametrize("error_type", [RuntimeError, RecursionError]) +def test_hostile_mapping_edge_is_bounded_domain_error(error_type): + from collections.abc import Mapping + + class ExplodingEdge(Mapping): + def __getitem__(self, key): + raise error_type("RAW-MAPPING-SENTINEL") + + def __iter__(self): + return iter(("from",)) + + def __len__(self): + return 1 + + def items(self): + raise error_type("RAW-MAPPING-SENTINEL") + + document = perseus.ingest_sbom_document(_load("spdx-app.json"), source_ref="artifact:hostile-edge") + with pytest.raises(perseus.SBOMLineageError) as exc_info: + perseus.build_sbom_lineage([document], edges=[ExplodingEdge()]) + assert "RAW-MAPPING-SENTINEL" not in str(exc_info.value) + + +@pytest.mark.parametrize("error_type", [RuntimeError, RecursionError]) +def test_hostile_normalized_document_mapping_is_bounded_domain_error(error_type): + from collections.abc import Mapping + + class ExplodingDocument(Mapping): + def __getitem__(self, key): + raise error_type("RAW-MAPPING-SENTINEL") + + def __iter__(self): + return iter(("schema_version",)) + + def __len__(self): + return 1 + + def items(self): + raise error_type("RAW-MAPPING-SENTINEL") + + with pytest.raises(perseus.SBOMLineageError) as exc_info: + perseus.build_sbom_lineage([ExplodingDocument()]) + assert "RAW-MAPPING-SENTINEL" not in str(exc_info.value) + result = perseus.verify_sbom_document(ExplodingDocument()) + assert result["valid"] is False + assert "RAW-MAPPING-SENTINEL" not in json.dumps(result) + + def test_bare_private_ip_in_non_purl_query_is_not_persisted(): payload = json.loads(_load("cyclonedx-app.json")) payload["components"][0]["externalReferences"] = [ From c7f40fd3f93fff21d9c8b4c0764e8a9e61924c52 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 13:08:49 +0000 Subject: [PATCH 26/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 6a2b9d12..b84c71ca 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "686c14a-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "d49a638" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 1aed8c5c2cd34da4ba1c529b68be48d66092b03a Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 13:31:02 +0000 Subject: [PATCH 27/40] fix(sbom): suppress hostile errors and validate format types --- README.md | 2 +- perseus.py | 8 +++---- src/perseus/sbom_lineage.py | 6 ++---- tests/test_sbom_lineage.py | 42 +++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index c90895e7..e8602cc9 100755 --- a/README.md +++ b/README.md @@ -274,7 +274,7 @@ Published as [`io.github.Perseus-Computing-LLC/perseus`](https://registry.modelc ### MCP Tools - + MCP tools resolve live state at invocation time, including the canonical Perseus Vault tool. Two additional sensitive tools — `perseus_query` (run a shell command) and `perseus_agent` (execute a local agent subprocess) — are **not** part of this default set: they require explicit `mcp.tool_allowlist` opt-in because they execute commands in the user's local shell (**not sandboxed, full user permissions apply**). diff --git a/perseus.py b/perseus.py index b84c71ca..5e5f71b7 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "d49a638" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "c7f40fd-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -45457,7 +45457,7 @@ def _sl_reject_unbound_document_edges(relationships: list[Any], document_id: str def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: - if fmt not in _SL_FORMATS: + if not isinstance(fmt, str) or fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") if _sl_node_kind(document_id) != "document": raise SBOMLineageError("document ID must use the document namespace") @@ -46041,8 +46041,6 @@ def _sl_snapshot_mapping(value: Any, field: str) -> dict[str, Any]: raise SBOMLineageError(f"{field} must be an object") try: snapshot, _ = _sl_snapshot_json(value) - except SBOMLineageError: - raise except Exception: raise SBOMLineageError(f"{field} is not bounded canonical JSON") from None if not isinstance(snapshot, dict): @@ -46487,7 +46485,7 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non if provenance is None or provenance != (unsigned.get("document_sha256"), _sl_json(document)): raise SBOMLineageError("normalized SBOM is not bound to raw ingestion bytes/source digest") fmt = document.get("format") - if fmt not in _SL_FORMATS: + if not isinstance(fmt, str) or fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") spec_version = document.get("spec_version") if not isinstance(spec_version, str): diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 5e3cae2f..d032638e 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -1021,7 +1021,7 @@ def _sl_reject_unbound_document_edges(relationships: list[Any], document_id: str def _sl_finalize_document(*, fmt: str, spec_version: str, document_id: str, document_name: str, created_at: str, supplier: str, components: list[dict[str, Any]], relationships: list[dict[str, Any]], raw_bytes: bytes, source_ref: str, truncated: list[str] | None = None, metadata_component_id: str = "") -> dict[str, Any]: - if fmt not in _SL_FORMATS: + if not isinstance(fmt, str) or fmt not in _SL_FORMATS: raise SBOMLineageError("unsupported SBOM format") if _sl_node_kind(document_id) != "document": raise SBOMLineageError("document ID must use the document namespace") @@ -1605,8 +1605,6 @@ def _sl_snapshot_mapping(value: Any, field: str) -> dict[str, Any]: raise SBOMLineageError(f"{field} must be an object") try: snapshot, _ = _sl_snapshot_json(value) - except SBOMLineageError: - raise except Exception: raise SBOMLineageError(f"{field} is not bounded canonical JSON") from None if not isinstance(snapshot, dict): @@ -2051,7 +2049,7 @@ def _sl_validate_document(document: Mapping[str, Any], *, raw_bytes: bytes | Non if provenance is None or provenance != (unsigned.get("document_sha256"), _sl_json(document)): raise SBOMLineageError("normalized SBOM is not bound to raw ingestion bytes/source digest") fmt = document.get("format") - if fmt not in _SL_FORMATS: + if not isinstance(fmt, str) or fmt not in _SL_FORMATS: raise SBOMLineageError("normalized SBOM format is invalid") spec_version = document.get("spec_version") if not isinstance(spec_version, str): diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index e55963f7..deaab0c3 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1852,6 +1852,48 @@ def items(self): assert "RAW-MAPPING-SENTINEL" not in json.dumps(result) +def test_attacker_domain_errors_are_not_echoed_by_public_boundaries(): + from collections.abc import Mapping + + class ExplodingMapping(Mapping): + def __getitem__(self, key): + raise perseus.SBOMLineageError("RAW-DOMAIN-SENTINEL") + + def __iter__(self): + return iter(("schema_version",)) + + def __len__(self): + return 1 + + def items(self): + raise perseus.SBOMLineageError("RAW-DOMAIN-SENTINEL") + + with pytest.raises(perseus.SBOMLineageError) as exc_info: + perseus.build_sbom_lineage([ExplodingMapping()]) + assert "RAW-DOMAIN-SENTINEL" not in str(exc_info.value) + for result in ( + perseus.verify_sbom_document(ExplodingMapping()), + perseus.verify_sbom_lineage(ExplodingMapping()), + perseus.verify_sbom_lineage_query(ExplodingMapping()), + ): + assert result["valid"] is False + assert "RAW-DOMAIN-SENTINEL" not in json.dumps(result) + + +def test_malformed_normalized_format_is_a_bounded_domain_error(): + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:malformed-format") + document["format"] = [] + unsigned = dict(document) + unsigned.pop("ingestion_digest") + document["ingestion_digest"] = perseus._sl_ingestion_sha(unsigned, raw) + with pytest.raises(perseus.SBOMLineageError, match="format"): + perseus.build_sbom_lineage([document], raw_documents=[raw]) + result = perseus.verify_sbom_document(document, raw_document=raw) + assert result["valid"] is False + assert "TypeError" not in json.dumps(result) + + def test_bare_private_ip_in_non_purl_query_is_not_persisted(): payload = json.loads(_load("cyclonedx-app.json")) payload["components"][0]["externalReferences"] = [ From 488d96efcb346ab61d1d029a582b6bb09af56b32 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 13:31:14 +0000 Subject: [PATCH 28/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 5e5f71b7..4fd817b6 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "c7f40fd-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "1aed8c5" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From e48af2dd829ac8551fa64aeb7f74a6568b9b0d0e Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 13:49:23 +0000 Subject: [PATCH 29/40] fix(sbom): sanitize raw mapping and edge iterator failures --- perseus.py | 52 ++++++++++++++++++++++++++----------- src/perseus/sbom_lineage.py | 50 +++++++++++++++++++++++++---------- tests/test_sbom_lineage.py | 52 +++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 29 deletions(-) diff --git a/perseus.py b/perseus.py index 4fd817b6..fed4ce05 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "1aed8c5" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "488d96e-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -45988,7 +45988,22 @@ def _sl_snapshot_json( snapshot: dict[str, Any] = {} size = 2 try: - for index, (key, item) in enumerate(value.items()): + try: + item_iterator = iter(value.items()) + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None + index = 0 + while True: + try: + pair = next(item_iterator) + except StopIteration: + break + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None + try: + key, item = pair + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None if not isinstance(key, str): raise SBOMLineageError("SBOM JSON object keys must be strings") key_text = str.__str__(key) @@ -45999,6 +46014,7 @@ def _sl_snapshot_json( if size > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") snapshot[key_text] = child + index += 1 finally: active.remove(marker) return snapshot, size @@ -46041,6 +46057,8 @@ def _sl_snapshot_mapping(value: Any, field: str) -> dict[str, Any]: raise SBOMLineageError(f"{field} must be an object") try: snapshot, _ = _sl_snapshot_json(value) + except SBOMLineageError: + raise except Exception: raise SBOMLineageError(f"{field} is not bounded canonical JSON") from None if not isinstance(snapshot, dict): @@ -46178,8 +46196,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise - except Exception as exc: - raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from exc + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None elif isinstance(document, str): bounded_text = _sl_bounded_text_bytes(document) possible_path = Path(document) @@ -46564,30 +46582,34 @@ def _sl_bounded_external_edges(edges: Any) -> list[Any]: return [] if isinstance(edges, (list, tuple)): try: - if len(edges) > _SL_MAX_EDGES: - raise SBOMLineageError("lineage edges exceed their bound") - except SBOMLineageError: - raise + edge_count = len(edges) except Exception: raise SBOMLineageError("lineage edge collection is invalid") from None + if edge_count > _SL_MAX_EDGES: + raise SBOMLineageError("lineage edges exceed their bound") if isinstance(edges, (str, bytes, bytearray, memoryview, Mapping)): raise SBOMLineageError("lineage edges must be a bounded collection") try: iterator = iter(edges) except Exception: raise SBOMLineageError("lineage edge collection is invalid") from None - result: list[Any] = [] + raw_items: list[Any] = [] + too_many = False try: for index, raw in enumerate(iterator): if index >= _SL_MAX_EDGES: - raise SBOMLineageError("lineage edges exceed their bound") - if not isinstance(raw, Mapping): - raise SBOMLineageError("lineage edges must contain objects") - result.append(_sl_snapshot_mapping(raw, "lineage edge")) - except SBOMLineageError: - raise + too_many = True + break + raw_items.append(raw) except Exception: raise SBOMLineageError("lineage edge collection is invalid") from None + if too_many: + raise SBOMLineageError("lineage edges exceed their bound") + result: list[Any] = [] + for raw in raw_items: + if not isinstance(raw, Mapping): + raise SBOMLineageError("lineage edges must contain objects") + result.append(_sl_snapshot_mapping(raw, "lineage edge")) return result diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index d032638e..e4d626fd 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -1552,7 +1552,22 @@ def _sl_snapshot_json( snapshot: dict[str, Any] = {} size = 2 try: - for index, (key, item) in enumerate(value.items()): + try: + item_iterator = iter(value.items()) + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None + index = 0 + while True: + try: + pair = next(item_iterator) + except StopIteration: + break + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None + try: + key, item = pair + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None if not isinstance(key, str): raise SBOMLineageError("SBOM JSON object keys must be strings") key_text = str.__str__(key) @@ -1563,6 +1578,7 @@ def _sl_snapshot_json( if size > _SL_MAX_INPUT_BYTES: raise SBOMLineageError(f"SBOM input exceeds {_SL_MAX_INPUT_BYTES} bytes") snapshot[key_text] = child + index += 1 finally: active.remove(marker) return snapshot, size @@ -1605,6 +1621,8 @@ def _sl_snapshot_mapping(value: Any, field: str) -> dict[str, Any]: raise SBOMLineageError(f"{field} must be an object") try: snapshot, _ = _sl_snapshot_json(value) + except SBOMLineageError: + raise except Exception: raise SBOMLineageError(f"{field} is not bounded canonical JSON") from None if not isinstance(snapshot, dict): @@ -1742,8 +1760,8 @@ def _sl_payload(document: Any) -> tuple[Any, bytes]: raw_bytes = _sl_json(snapshot).encode("utf-8") except SBOMLineageError: raise - except Exception as exc: - raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from exc + except Exception: + raise SBOMLineageError("SBOM mapping is not bounded canonical JSON") from None elif isinstance(document, str): bounded_text = _sl_bounded_text_bytes(document) possible_path = Path(document) @@ -2128,30 +2146,34 @@ def _sl_bounded_external_edges(edges: Any) -> list[Any]: return [] if isinstance(edges, (list, tuple)): try: - if len(edges) > _SL_MAX_EDGES: - raise SBOMLineageError("lineage edges exceed their bound") - except SBOMLineageError: - raise + edge_count = len(edges) except Exception: raise SBOMLineageError("lineage edge collection is invalid") from None + if edge_count > _SL_MAX_EDGES: + raise SBOMLineageError("lineage edges exceed their bound") if isinstance(edges, (str, bytes, bytearray, memoryview, Mapping)): raise SBOMLineageError("lineage edges must be a bounded collection") try: iterator = iter(edges) except Exception: raise SBOMLineageError("lineage edge collection is invalid") from None - result: list[Any] = [] + raw_items: list[Any] = [] + too_many = False try: for index, raw in enumerate(iterator): if index >= _SL_MAX_EDGES: - raise SBOMLineageError("lineage edges exceed their bound") - if not isinstance(raw, Mapping): - raise SBOMLineageError("lineage edges must contain objects") - result.append(_sl_snapshot_mapping(raw, "lineage edge")) - except SBOMLineageError: - raise + too_many = True + break + raw_items.append(raw) except Exception: raise SBOMLineageError("lineage edge collection is invalid") from None + if too_many: + raise SBOMLineageError("lineage edges exceed their bound") + result: list[Any] = [] + for raw in raw_items: + if not isinstance(raw, Mapping): + raise SBOMLineageError("lineage edges must contain objects") + result.append(_sl_snapshot_mapping(raw, "lineage edge")) return result diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index deaab0c3..778d55c3 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1852,6 +1852,58 @@ def items(self): assert "RAW-MAPPING-SENTINEL" not in json.dumps(result) +def test_hostile_raw_mapping_errors_are_not_echoed_at_rebinding_boundaries(): + from collections.abc import Mapping + + class ExplodingRaw(Mapping): + def __getitem__(self, key): + raise perseus.SBOMLineageError("RAW-ARTIFACT-SENTINEL") + + def __iter__(self): + return iter(("spdxVersion",)) + + def __len__(self): + return 1 + + def items(self): + raise perseus.SBOMLineageError("RAW-ARTIFACT-SENTINEL") + + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:raw-boundary") + lineage = perseus.build_sbom_lineage([document], edges=[], raw_documents=[raw]) + query = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + with pytest.raises(perseus.SBOMLineageError) as exc_info: + perseus.build_sbom_lineage([document], edges=[], raw_documents=[ExplodingRaw()]) + assert "RAW-ARTIFACT-SENTINEL" not in str(exc_info.value) + for result in ( + perseus.verify_sbom_document(document, raw_document=ExplodingRaw()), + perseus.verify_sbom_lineage(lineage, raw_documents=[ExplodingRaw()]), + perseus.verify_sbom_lineage_query(query, lineage, [ExplodingRaw()]), + ): + assert result["valid"] is False + assert "RAW-ARTIFACT-SENTINEL" not in json.dumps(result) + + +def test_hostile_edge_sequence_errors_are_not_echoed(): + class ExplodingEdges(list): + def __len__(self): + raise perseus.SBOMLineageError("RAW-LIST-SENTINEL") + + raw = _load("spdx-app.json") + document = perseus.ingest_sbom_document(raw, source_ref="artifact:sequence-boundary") + lineage = perseus.build_sbom_lineage([document], edges=[], raw_documents=[raw]) + query = perseus.query_sbom_lineage(lineage, "CVE-2021-44228") + with pytest.raises(perseus.SBOMLineageError) as exc_info: + perseus.build_sbom_lineage([document], edges=ExplodingEdges()) + assert "RAW-LIST-SENTINEL" not in str(exc_info.value) + for result in ( + perseus.verify_sbom_lineage(lineage, raw_documents=[raw], edges=ExplodingEdges()), + perseus.verify_sbom_lineage_query(query, lineage, [raw], edges=ExplodingEdges()), + ): + assert result["valid"] is False + assert "RAW-LIST-SENTINEL" not in json.dumps(result) + + def test_attacker_domain_errors_are_not_echoed_by_public_boundaries(): from collections.abc import Mapping From 4c2967839210bf5ef126d8e74e3e620e4e50ab41 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 13:49:36 +0000 Subject: [PATCH 30/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index fed4ce05..4befbefd 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "488d96e-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "e48af2d" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 9be87a58fac3dffa5ee3ff7ae9903b2cee7864d0 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 19:25:13 +0000 Subject: [PATCH 31/40] fix(sbom): require authoritative XML namespaces --- perseus.py | 18 ++++++++++++++---- src/perseus/sbom_lineage.py | 16 +++++++++++++--- tests/test_sbom_lineage.py | 11 +++++++++++ 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/perseus.py b/perseus.py index 4befbefd..9423b9b6 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "e48af2d" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "4c29678-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44461,6 +44461,8 @@ def cmd_code_map(args, cfg) -> int: _SL_LINEAGE_SCHEMA = "perseus-software-lineage/v1" _SL_QUERY_SCHEMA = "perseus-software-lineage-query/v1" _SL_FORMATS = frozenset({"SPDX", "CycloneDX"}) +_SL_SPDX_XML_NAMESPACE = "http://spdx.org/rdf/terms#" +_SL_CDX_XML_NAMESPACES = frozenset(f"http://cyclonedx.org/schema/bom-{version}.xsd" for version in ("1.4", "1.5", "1.6")) _SL_SPDX_VERSIONS = frozenset({"2.2", "2.3"}) _SL_CDX_VERSIONS = frozenset({"1.4", "1.5", "1.6"}) _SL_CONFIDENCE = frozenset({"high", "medium", "low", "unknown"}) @@ -44997,15 +44999,22 @@ def _sl_validate_xml_discriminators(root: Any) -> None: families: set[str] = set() for element in root.iter(): local = _sl_local(element).casefold() - namespace = str(getattr(element, "tag", "")).split("}", 1)[0].casefold() - if local in spdx_markers or "spdx.org" in namespace: + namespace = _sl_xml_namespace(element).casefold() + if local in spdx_markers or namespace == _SL_SPDX_XML_NAMESPACE: families.add("SPDX") - if local in cdx_markers or "cyclonedx.org" in namespace: + if local in cdx_markers or namespace in _SL_CDX_XML_NAMESPACES: families.add("CycloneDX") if len(families) > 1: raise SBOMLineageError("conflicting XML SBOM format markers") +def _sl_validate_spdx_xml_namespace(root: Any) -> None: + for element in root.iter(): + namespace = _sl_xml_namespace(element) + if namespace and namespace.casefold() != _SL_SPDX_XML_NAMESPACE: + raise SBOMLineageError("SPDX XML namespace is not authoritative") + + def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) @@ -45714,6 +45723,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + _sl_validate_spdx_xml_namespace(root) version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) document_id = _sl_spdx_document_id(_sl_xml_text(root, "SPDXID")) creation = _sl_single_child(root, "creationInfo") diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index e4d626fd..14669006 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -25,6 +25,8 @@ _SL_LINEAGE_SCHEMA = "perseus-software-lineage/v1" _SL_QUERY_SCHEMA = "perseus-software-lineage-query/v1" _SL_FORMATS = frozenset({"SPDX", "CycloneDX"}) +_SL_SPDX_XML_NAMESPACE = "http://spdx.org/rdf/terms#" +_SL_CDX_XML_NAMESPACES = frozenset(f"http://cyclonedx.org/schema/bom-{version}.xsd" for version in ("1.4", "1.5", "1.6")) _SL_SPDX_VERSIONS = frozenset({"2.2", "2.3"}) _SL_CDX_VERSIONS = frozenset({"1.4", "1.5", "1.6"}) _SL_CONFIDENCE = frozenset({"high", "medium", "low", "unknown"}) @@ -561,15 +563,22 @@ def _sl_validate_xml_discriminators(root: Any) -> None: families: set[str] = set() for element in root.iter(): local = _sl_local(element).casefold() - namespace = str(getattr(element, "tag", "")).split("}", 1)[0].casefold() - if local in spdx_markers or "spdx.org" in namespace: + namespace = _sl_xml_namespace(element).casefold() + if local in spdx_markers or namespace == _SL_SPDX_XML_NAMESPACE: families.add("SPDX") - if local in cdx_markers or "cyclonedx.org" in namespace: + if local in cdx_markers or namespace in _SL_CDX_XML_NAMESPACES: families.add("CycloneDX") if len(families) > 1: raise SBOMLineageError("conflicting XML SBOM format markers") +def _sl_validate_spdx_xml_namespace(root: Any) -> None: + for element in root.iter(): + namespace = _sl_xml_namespace(element) + if namespace and namespace.casefold() != _SL_SPDX_XML_NAMESPACE: + raise SBOMLineageError("SPDX XML namespace is not authoritative") + + def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) @@ -1278,6 +1287,7 @@ def _sl_spdx_rdf_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, def _sl_spdx_xml(root: Any, raw_bytes: bytes, source_ref: str) -> dict[str, Any]: + _sl_validate_spdx_xml_namespace(root) version = _sl_validate_spdx_version(_sl_xml_text(root, "spdxVersion")) document_id = _sl_spdx_document_id(_sl_xml_text(root, "SPDXID")) creation = _sl_single_child(root, "creationInfo") diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 778d55c3..3b5abe4b 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1978,6 +1978,17 @@ def test_rdf_parent_about_conflicting_child_spdxid_fails_closed(): perseus._sl_spdx_rdf_xml(root, b"", "artifact:rdf-conflict") +def test_spdx_xml_rejects_non_authoritative_namespace_with_official_substring(): + xml = ( + '' + 'SPDX-2.3SPDXRef-DOCUMENT' + '2024-01-01T00:00:00Z' + 'Tool: test' + ) + with pytest.raises(perseus.SBOMLineageError, match="namespace"): + perseus.ingest_sbom_document(xml) + + def test_cyclonedx_xml_rejects_attacker_namespace_child(): xml = ( ' Date: Thu, 20 Aug 2026 19:25:26 +0000 Subject: [PATCH 32/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 9423b9b6..6f51d6e9 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "4c29678-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "9be87a5" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 12ccbfa8436fb73affaadfb5be0e73179a3c720a Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 19:41:33 +0000 Subject: [PATCH 33/40] fix(sbom): reject case-variant XML namespaces --- perseus.py | 9 ++++++--- src/perseus/sbom_lineage.py | 7 +++++-- tests/test_sbom_lineage.py | 8 ++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/perseus.py b/perseus.py index 6f51d6e9..d65e4a75 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "9be87a5" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "aea92ff-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44999,7 +44999,7 @@ def _sl_validate_xml_discriminators(root: Any) -> None: families: set[str] = set() for element in root.iter(): local = _sl_local(element).casefold() - namespace = _sl_xml_namespace(element).casefold() + namespace = _sl_xml_namespace(element) if local in spdx_markers or namespace == _SL_SPDX_XML_NAMESPACE: families.add("SPDX") if local in cdx_markers or namespace in _SL_CDX_XML_NAMESPACES: @@ -45011,7 +45011,7 @@ def _sl_validate_xml_discriminators(root: Any) -> None: def _sl_validate_spdx_xml_namespace(root: Any) -> None: for element in root.iter(): namespace = _sl_xml_namespace(element) - if namespace and namespace.casefold() != _SL_SPDX_XML_NAMESPACE: + if namespace and namespace != _SL_SPDX_XML_NAMESPACE: raise SBOMLineageError("SPDX XML namespace is not authoritative") @@ -46260,6 +46260,9 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An _sl_validate_xml_discriminators(value) root_name = _sl_local(value).casefold() if root_name == "spdxdocument": + root_namespace = _sl_xml_namespace(value) + if root_namespace and root_namespace != _SL_SPDX_XML_NAMESPACE: + raise SBOMLineageError("SPDX XML namespace is not authoritative") if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) return _sl_spdx_xml(value, raw_bytes, normalized_source) diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 14669006..1270189d 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -563,7 +563,7 @@ def _sl_validate_xml_discriminators(root: Any) -> None: families: set[str] = set() for element in root.iter(): local = _sl_local(element).casefold() - namespace = _sl_xml_namespace(element).casefold() + namespace = _sl_xml_namespace(element) if local in spdx_markers or namespace == _SL_SPDX_XML_NAMESPACE: families.add("SPDX") if local in cdx_markers or namespace in _SL_CDX_XML_NAMESPACES: @@ -575,7 +575,7 @@ def _sl_validate_xml_discriminators(root: Any) -> None: def _sl_validate_spdx_xml_namespace(root: Any) -> None: for element in root.iter(): namespace = _sl_xml_namespace(element) - if namespace and namespace.casefold() != _SL_SPDX_XML_NAMESPACE: + if namespace and namespace != _SL_SPDX_XML_NAMESPACE: raise SBOMLineageError("SPDX XML namespace is not authoritative") @@ -1824,6 +1824,9 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An _sl_validate_xml_discriminators(value) root_name = _sl_local(value).casefold() if root_name == "spdxdocument": + root_namespace = _sl_xml_namespace(value) + if root_namespace and root_namespace != _SL_SPDX_XML_NAMESPACE: + raise SBOMLineageError("SPDX XML namespace is not authoritative") if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) return _sl_spdx_xml(value, raw_bytes, normalized_source) diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 3b5abe4b..91779d43 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1978,6 +1978,14 @@ def test_rdf_parent_about_conflicting_child_spdxid_fails_closed(): perseus._sl_spdx_rdf_xml(root, b"", "artifact:rdf-conflict") +def test_spdx_xml_rejects_case_variant_namespace(): + xml = _load("spdx-app.xml").replace( + b"http://spdx.org/rdf/terms#", b"HTTP://SPDX.ORG/RDF/TERMS#", + ) + with pytest.raises(perseus.SBOMLineageError, match="namespace"): + perseus.ingest_sbom_document(xml) + + def test_spdx_xml_rejects_non_authoritative_namespace_with_official_substring(): xml = ( '' From ab0ba12c0050b765745d822958b1f23ca45246b9 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 19:41:40 +0000 Subject: [PATCH 34/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index d65e4a75..6594810d 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "aea92ff-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "12ccbfa" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 62a9fad88d1ec8fa9edfe2a16bef95db3010f1f2 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 19:56:20 +0000 Subject: [PATCH 35/40] fix(sbom): gate RDF namespaces before parser selection --- perseus.py | 18 ++++++++++++++---- src/perseus/sbom_lineage.py | 16 +++++++++++++--- tests/test_sbom_lineage.py | 20 ++++++++++++++++++++ 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/perseus.py b/perseus.py index 6594810d..35f8fe84 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "12ccbfa" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "ab0ba12-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -44462,6 +44462,7 @@ def cmd_code_map(args, cfg) -> int: _SL_QUERY_SCHEMA = "perseus-software-lineage-query/v1" _SL_FORMATS = frozenset({"SPDX", "CycloneDX"}) _SL_SPDX_XML_NAMESPACE = "http://spdx.org/rdf/terms#" +_SL_RDF_XML_NAMESPACE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" _SL_CDX_XML_NAMESPACES = frozenset(f"http://cyclonedx.org/schema/bom-{version}.xsd" for version in ("1.4", "1.5", "1.6")) _SL_SPDX_VERSIONS = frozenset({"2.2", "2.3"}) _SL_CDX_VERSIONS = frozenset({"1.4", "1.5", "1.6"}) @@ -45015,6 +45016,16 @@ def _sl_validate_spdx_xml_namespace(root: Any) -> None: raise SBOMLineageError("SPDX XML namespace is not authoritative") +def _sl_validate_spdx_rdf_xml_namespace(root: Any) -> None: + if _sl_xml_namespace(root) != _SL_RDF_XML_NAMESPACE: + raise SBOMLineageError("RDF XML namespace is not authoritative") + allowed = {_SL_SPDX_XML_NAMESPACE, _SL_RDF_XML_NAMESPACE} + for element in root.iter(): + namespace = _sl_xml_namespace(element) + if namespace and namespace not in allowed: + raise SBOMLineageError("RDF XML namespace is not authoritative") + + def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) @@ -46260,13 +46271,12 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An _sl_validate_xml_discriminators(value) root_name = _sl_local(value).casefold() if root_name == "spdxdocument": - root_namespace = _sl_xml_namespace(value) - if root_namespace and root_namespace != _SL_SPDX_XML_NAMESPACE: - raise SBOMLineageError("SPDX XML namespace is not authoritative") + _sl_validate_spdx_xml_namespace(value) if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) return _sl_spdx_xml(value, raw_bytes, normalized_source) if root_name == "rdf" and _sl_descendants(value, "SpdxDocument"): + _sl_validate_spdx_rdf_xml_namespace(value) return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) if root_name == "bom": return _sl_parse_cdx_xml(value, raw_bytes, normalized_source) diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 1270189d..8de299cd 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -26,6 +26,7 @@ _SL_QUERY_SCHEMA = "perseus-software-lineage-query/v1" _SL_FORMATS = frozenset({"SPDX", "CycloneDX"}) _SL_SPDX_XML_NAMESPACE = "http://spdx.org/rdf/terms#" +_SL_RDF_XML_NAMESPACE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" _SL_CDX_XML_NAMESPACES = frozenset(f"http://cyclonedx.org/schema/bom-{version}.xsd" for version in ("1.4", "1.5", "1.6")) _SL_SPDX_VERSIONS = frozenset({"2.2", "2.3"}) _SL_CDX_VERSIONS = frozenset({"1.4", "1.5", "1.6"}) @@ -579,6 +580,16 @@ def _sl_validate_spdx_xml_namespace(root: Any) -> None: raise SBOMLineageError("SPDX XML namespace is not authoritative") +def _sl_validate_spdx_rdf_xml_namespace(root: Any) -> None: + if _sl_xml_namespace(root) != _SL_RDF_XML_NAMESPACE: + raise SBOMLineageError("RDF XML namespace is not authoritative") + allowed = {_SL_SPDX_XML_NAMESPACE, _SL_RDF_XML_NAMESPACE} + for element in root.iter(): + namespace = _sl_xml_namespace(element) + if namespace and namespace not in allowed: + raise SBOMLineageError("RDF XML namespace is not authoritative") + + def _sl_xml_value(element: Any, *names: str, default: str = "") -> str: return _sl_xml_scalar(element, tuple(names), default=default) @@ -1824,13 +1835,12 @@ def ingest_sbom_document(document: Any, *, source_ref: str = "") -> dict[str, An _sl_validate_xml_discriminators(value) root_name = _sl_local(value).casefold() if root_name == "spdxdocument": - root_namespace = _sl_xml_namespace(value) - if root_namespace and root_namespace != _SL_SPDX_XML_NAMESPACE: - raise SBOMLineageError("SPDX XML namespace is not authoritative") + _sl_validate_spdx_xml_namespace(value) if _sl_descendants(value, "Package") or _sl_descendants(value, "Relationship") and not _sl_children(value, "package"): return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) return _sl_spdx_xml(value, raw_bytes, normalized_source) if root_name == "rdf" and _sl_descendants(value, "SpdxDocument"): + _sl_validate_spdx_rdf_xml_namespace(value) return _sl_spdx_rdf_xml(value, raw_bytes, normalized_source) if root_name == "bom": return _sl_parse_cdx_xml(value, raw_bytes, normalized_source) diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 91779d43..1c3a6c66 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1978,6 +1978,26 @@ def test_rdf_parent_about_conflicting_child_spdxid_fails_closed(): perseus._sl_spdx_rdf_xml(root, b"", "artifact:rdf-conflict") +def test_spdx_xml_rejects_foreign_child_namespace_before_rdf_selection(): + xml = _load("spdx-app.xml").replace( + b"", b'', + ).replace(b"", b"") + with pytest.raises(perseus.SBOMLineageError, match="namespace"): + perseus.ingest_sbom_document(xml) + + +def test_spdx_rdf_rejects_foreign_and_case_variant_spdx_namespaces(): + for namespace in ( + b"HTTP://SPDX.ORG/RDF/TERMS#", + b"https://attacker.example/spdx.org/rdf/terms#", + ): + xml = _load("spdx-rdf.xml").replace( + b"http://spdx.org/rdf/terms#", namespace, + ) + with pytest.raises(perseus.SBOMLineageError, match="namespace"): + perseus.ingest_sbom_document(xml) + + def test_spdx_xml_rejects_case_variant_namespace(): xml = _load("spdx-app.xml").replace( b"http://spdx.org/rdf/terms#", b"HTTP://SPDX.ORG/RDF/TERMS#", From 7919d4fbc133fa19a8f9f39354aa5279b8a166ed Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 19:56:27 +0000 Subject: [PATCH 36/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 35f8fe84..91b28792 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "ab0ba12-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "62a9fad" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 5aca12aa5254eb33225bb4d1e9d0158e2206b230 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 20:29:44 +0000 Subject: [PATCH 37/40] fix(sbom): preserve raw bytes on Windows readers --- perseus.py | 4 ++-- src/perseus/sbom_lineage.py | 2 +- tests/test_sbom_lineage.py | 21 ++++++++++++++++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/perseus.py b/perseus.py index 91b28792..6f8a9d7c 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "62a9fad" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "7919d4f-dirty" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: @@ -45900,7 +45900,7 @@ def _sl_read_bounded(path: Path) -> bytes: initial = _sl_os.lstat(path) if not _sl_stat.S_ISREG(initial.st_mode): raise SBOMLineageError("SBOM input must be a regular file") - flags = _sl_os.O_RDONLY | getattr(_sl_os, "O_CLOEXEC", 0) | getattr(_sl_os, "O_NOFOLLOW", 0) | getattr(_sl_os, "O_NONBLOCK", 0) + flags = _sl_os.O_RDONLY | getattr(_sl_os, "O_BINARY", 0) | getattr(_sl_os, "O_CLOEXEC", 0) | getattr(_sl_os, "O_NOFOLLOW", 0) | getattr(_sl_os, "O_NONBLOCK", 0) fd = _sl_os.open(path, flags) except SBOMLineageError: raise diff --git a/src/perseus/sbom_lineage.py b/src/perseus/sbom_lineage.py index 8de299cd..0dbf2b7d 100644 --- a/src/perseus/sbom_lineage.py +++ b/src/perseus/sbom_lineage.py @@ -1464,7 +1464,7 @@ def _sl_read_bounded(path: Path) -> bytes: initial = _sl_os.lstat(path) if not _sl_stat.S_ISREG(initial.st_mode): raise SBOMLineageError("SBOM input must be a regular file") - flags = _sl_os.O_RDONLY | getattr(_sl_os, "O_CLOEXEC", 0) | getattr(_sl_os, "O_NOFOLLOW", 0) | getattr(_sl_os, "O_NONBLOCK", 0) + flags = _sl_os.O_RDONLY | getattr(_sl_os, "O_BINARY", 0) | getattr(_sl_os, "O_CLOEXEC", 0) | getattr(_sl_os, "O_NOFOLLOW", 0) | getattr(_sl_os, "O_NONBLOCK", 0) fd = _sl_os.open(path, flags) except SBOMLineageError: raise diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 1c3a6c66..46e1b133 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1145,7 +1145,26 @@ def unexpected_read(*args, **kwargs): assert perseus._sl_read_bounded(path) == b"{}" +def test_bounded_reader_requests_binary_mode_for_cross_platform_hashes(tmp_path, monkeypatch): + path = tmp_path / "binary.json" + path.write_bytes(b"{\"value\":1}\r\n") + binary_flag = getattr(perseus._sl_os, "O_BINARY", 0x8000) + observed = [] + real_open = perseus._sl_os.open + monkeypatch.setattr(perseus._sl_os, "O_BINARY", binary_flag, raising=False) + + def checked_open(file_path, flags): + observed.append(flags) + return real_open(file_path, flags & ~binary_flag) + + monkeypatch.setattr(perseus._sl_os, "open", checked_open) + assert perseus._sl_read_bounded(path) == b"{\"value\":1}\r\n" + assert observed and observed[0] & binary_flag + + def test_bounded_reader_rejects_non_regular_inputs_before_opening(tmp_path, monkeypatch): + if not hasattr(os, "mkfifo"): + pytest.skip("named pipes are not available on this platform") path = tmp_path / "special" os.mkfifo(path) monkeypatch.setattr(Path, "read_bytes", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("special file was opened"))) @@ -1700,7 +1719,7 @@ def test_cyclonedx_xml_properties_are_retained_in_complete_coverage(): " CVE-XML-PROPERTY\n" " Apache-2.0" ) - xml = _load("cyclonedx-app.xml").decode("utf-8").replace( + xml = _load("cyclonedx-app.xml").decode("utf-8").replace("\r\n", "\n").replace( "2026-08-19T12:00:00Z", "2026-08-19T12:00:00Zperseus-build-xml", ) From 25e03feeef2b05df8bf34310c7146c4917582656 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 20:29:53 +0000 Subject: [PATCH 38/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 6f8a9d7c..8e7b833e 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "7919d4f-dirty" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "5aca12a" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: From 994edcad225f2e688b89416ca118d2e2d15acb81 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 20:49:11 +0000 Subject: [PATCH 39/40] test(sbom): preserve native binary flag in portability probe --- tests/test_sbom_lineage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_sbom_lineage.py b/tests/test_sbom_lineage.py index 46e1b133..1e38fda6 100644 --- a/tests/test_sbom_lineage.py +++ b/tests/test_sbom_lineage.py @@ -1149,13 +1149,16 @@ def test_bounded_reader_requests_binary_mode_for_cross_platform_hashes(tmp_path, path = tmp_path / "binary.json" path.write_bytes(b"{\"value\":1}\r\n") binary_flag = getattr(perseus._sl_os, "O_BINARY", 0x8000) + actual_binary_flag = getattr(os, "O_BINARY", None) observed = [] real_open = perseus._sl_os.open monkeypatch.setattr(perseus._sl_os, "O_BINARY", binary_flag, raising=False) def checked_open(file_path, flags): observed.append(flags) - return real_open(file_path, flags & ~binary_flag) + if actual_binary_flag is None: + flags &= ~binary_flag + return real_open(file_path, flags) monkeypatch.setattr(perseus._sl_os, "open", checked_open) assert perseus._sl_read_bounded(path) == b"{\"value\":1}\r\n" From 761ca6e78607a3fcf88a656bd3c999c3545ea3f1 Mon Sep 17 00:00:00 2001 From: Hermes Research Date: Thu, 20 Aug 2026 20:49:18 +0000 Subject: [PATCH 40/40] build: refresh SBOM artifact provenance --- perseus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/perseus.py b/perseus.py index 8e7b833e..fdbacd66 100644 --- a/perseus.py +++ b/perseus.py @@ -79,7 +79,7 @@ def _perseus_lazy_urllib_request(name, _urllib=urllib): # ── Build provenance (injected by scripts/build.py at build time) ─────────── # Short git SHA of the source revision the artifact was built from (#853). # Empty when unknown (unbuilt source tree without git metadata). -_PERSEUS_BUILD_SHA = "5aca12a" # replaced at build time by scripts/build.py — see #853 +_PERSEUS_BUILD_SHA = "994edca" # replaced at build time by scripts/build.py — see #853 def _perseus_build_sha() -> str: