diff --git a/.buildkite/hooks/post-checkout b/.buildkite/hooks/post-checkout index b9f2828e9..61b05791d 100644 --- a/.buildkite/hooks/post-checkout +++ b/.buildkite/hooks/post-checkout @@ -48,6 +48,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == ml-cpp* ]]; then fi fi + if [[ "$BUILDKITE_STEP_KEY" == "snyk_sbom_scan" ]]; then + SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-ml-cpp/snyk/api_token 2>/dev/null || echo "") + if [ -n "$SNYK_TOKEN" ]; then + export SNYK_TOKEN + fi + fi + if [[ "$BUILDKITE_STEP_KEY" == "validate_pytorch_allowlist" ]]; then HF_TOKEN=$(vault read -field=token secret/ci/elastic-ml-cpp/huggingface/hf_token 2>/dev/null || echo "") if [ -n "$HF_TOKEN" ]; then diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 785485fb5..95655a0a4 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -125,6 +125,12 @@ def main(): ".buildkite/pipelines/check_build_regression.yml.sh", soft_fail=True)) + # Scan the shipped third-party libraries (Boost, libtorch, ...) with Snyk; the + # Snyk GitHub integration only sees the build-image Dockerfiles. + pipeline_steps.append(pipeline_steps.generate_step("Upload Snyk scan of shipped third-party libraries", + ".buildkite/pipelines/snyk_sbom_scan.yml.sh", + soft_fail=True)) + # Validate the PyTorch allowlist against HuggingFace models when # triggered from the PyTorch edge pipeline. Runs in a python:3 # container since the build/test images don't include Python. diff --git a/.buildkite/pipelines/snyk_sbom_scan.yml.sh b/.buildkite/pipelines/snyk_sbom_scan.yml.sh new file mode 100755 index 000000000..3cc72c808 --- /dev/null +++ b/.buildkite/pipelines/snyk_sbom_scan.yml.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Scan the third-party libraries that ship in the ml-cpp artifacts with Snyk. +# The Snyk GitHub integration only scans the Dockerfiles (build images), so the +# source-built libraries that actually ship (Boost, libtorch, ...) are checked +# here from an SBOM generated from 3rd_party/licenses/*INFO.csv. +# +# soft_fail while the baseline is being understood: findings turn the step +# yellow without failing the build. + +cat <<'EOL' +steps: + - label: "Snyk scan of shipped third-party libraries :snyk:" + key: "snyk_sbom_scan" + command: + - ".buildkite/scripts/steps/snyk-sbom-scan.sh" + soft_fail: true + agents: + image: "python:3.11-bookworm" + notify: + - github_commit_status: + context: "Snyk scan of shipped third-party libraries" +EOL diff --git a/.buildkite/scripts/steps/snyk-sbom-scan.sh b/.buildkite/scripts/steps/snyk-sbom-scan.sh new file mode 100755 index 000000000..0c6f635f6 --- /dev/null +++ b/.buildkite/scripts/steps/snyk-sbom-scan.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +# Generate an SBOM of the shipped third-party libraries and test it with Snyk. +# SNYK_TOKEN is injected by .buildkite/hooks/post-checkout for this step key. + +set -euo pipefail + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +cd "${REPO_ROOT}" + +SNYK_ORG="${SNYK_ORG:-machine-learning-48e}" +SNYK_SEVERITY_THRESHOLD="${SNYK_SEVERITY_THRESHOLD:-high}" +SBOM_FILE="${SBOM_FILE:-sbom.cdx.json}" + +python3 dev-tools/generate_sbom.py --output "${SBOM_FILE}" + +if [ -z "${SNYK_TOKEN:-}" ]; then + # Fail loudly rather than reporting a green scan that never ran. The step is + # soft_fail, so this shows up as a warning and does not block the build. + echo "ERROR: SNYK_TOKEN is not set; the Snyk scan did not run." >&2 + echo "Expected the post-checkout hook to read it from Vault for step key snyk_sbom_scan." >&2 + exit 1 +fi + +# Install the Snyk CLI and verify the published checksum before running it. +SNYK_BIN="$(mktemp -d)/snyk" +curl -fsSL -o "${SNYK_BIN}" https://static.snyk.io/cli/latest/snyk-linux +EXPECTED_SHA="$(curl -fsSL https://static.snyk.io/cli/latest/snyk-linux.sha256 | awk '{print $1}')" +echo "${EXPECTED_SHA} ${SNYK_BIN}" | sha256sum -c - +chmod +x "${SNYK_BIN}" + +# Non-zero exit when vulnerabilities at or above the threshold are found. +"${SNYK_BIN}" sbom test \ + --file="${SBOM_FILE}" \ + --org="${SNYK_ORG}" \ + --severity-threshold="${SNYK_SEVERITY_THRESHOLD}" diff --git a/dev-tools/generate_sbom.py b/dev-tools/generate_sbom.py new file mode 100755 index 000000000..4d181126e --- /dev/null +++ b/dev-tools/generate_sbom.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +"""Generate a CycloneDX SBOM of the third-party libraries ml-cpp ships. + +The Snyk integration on this repo scans the Dockerfiles, i.e. the build images, +which says nothing about what ends up in the released zip. The libraries that do +ship (Boost, libtorch, ...) are built from source, so Snyk cannot see them. + +This script turns the per-component ``3rd_party/licenses/*INFO.csv`` files (the +source of truth for shipped versions, also used by dependency_report.sh) into a +CycloneDX JSON document that ``snyk sbom test`` can analyse. + +Only components Snyk can match are emitted. Snyk resolves ``pkg:pypi/...`` through +its PyPI database and some ``pkg:generic/...`` names through its unmanaged C/C++ +database. A component Snyk does not recognise is silently ignored by the scan, so +those are reported on stderr instead of being emitted: a clean scan must not be +mistaken for a component having been checked. + +To cover another component, add it to ``_PURL_BY_NAME`` after confirming Snyk +matches it, e.g. by testing an old, known-vulnerable version. + +Usage: + generate_sbom.py [--licenses-dir DIR] [--output FILE] +""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +_REPO_ROOT = Path(__file__).resolve().parent.parent +_DEFAULT_LICENSES_DIR = _REPO_ROOT / "3rd_party" / "licenses" + +# INFO.csv component name (lowercased) -> (purl type and name, SBOM component name). +# Each entry has been confirmed to be matched by Snyk; see the module docstring. +_PURL_BY_NAME: Dict[str, Tuple[str, str]] = { + "boost c++ libraries": ("generic/boost", "boost"), + "libxml2": ("generic/libxml2", "libxml2"), + # libtorch is built from the PyTorch sources that the PyPI "torch" package ships. + "pytorch": ("pypi/torch", "torch"), +} + + +def _read_info_rows(licenses_dir: Path) -> List[Dict[str, str]]: + rows: List[Dict[str, str]] = [] + for info_file in sorted(licenses_dir.glob("*INFO.csv")): + with info_file.open(newline="") as handle: + file_rows = list(csv.DictReader(handle)) + # dependency_report.cmake only uses the last line of each file. + if file_rows: + rows.append(file_rows[-1]) + return rows + + +def build_sbom(licenses_dir: Path) -> Tuple[dict, List[str]]: + """Return (CycloneDX document, names of components that could not be emitted).""" + components: List[dict] = [] + skipped: List[str] = [] + + for row in _read_info_rows(licenses_dir): + name = row.get("name", "").strip() + version = row.get("version", "").strip() + mapping: Optional[Tuple[str, str]] = _PURL_BY_NAME.get(name.lower()) + if mapping is None or not version: + skipped.append(name) + continue + purl_path, component_name = mapping + purl = f"pkg:{purl_path}@{version}" + components.append( + { + "type": "library", + "name": component_name, + "version": version, + "purl": purl, + "bom-ref": purl, + } + ) + + sbom = { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": {"component": {"type": "application", "name": "ml-cpp", "version": "0"}}, + "components": components, + } + return sbom, skipped + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("--licenses-dir", type=Path, default=_DEFAULT_LICENSES_DIR) + parser.add_argument("--output", type=Path, default=Path("sbom.cdx.json")) + args = parser.parse_args(argv) + + sbom, skipped = build_sbom(args.licenses_dir) + + if skipped: + print( + "Not scanned (no Snyk-matchable purl mapping): " + ", ".join(skipped), + file=sys.stderr, + ) + if not sbom["components"]: + sys.exit("error: no scannable components found in " + str(args.licenses_dir)) + + args.output.write_text(json.dumps(sbom, indent=2) + "\n") + print( + "Wrote {} component(s) to {}: {}".format( + len(sbom["components"]), + args.output, + ", ".join(c["purl"] for c in sbom["components"]), + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev-tools/unittest/test_generate_sbom.py b/dev-tools/unittest/test_generate_sbom.py new file mode 100644 index 000000000..e31ceeac7 --- /dev/null +++ b/dev-tools/unittest/test_generate_sbom.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. + +"""Pytest tests for dev-tools/generate_sbom.py. + +Verifies the CycloneDX SBOM built from 3rd_party/licenses/*INFO.csv only lists +components that Snyk can match, and that components it cannot match are reported +rather than silently dropped. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import pytest + +_DEV_TOOLS = Path(__file__).resolve().parents[1] +if str(_DEV_TOOLS) not in sys.path: + sys.path.insert(0, str(_DEV_TOOLS)) + +import generate_sbom as gs # noqa: E402 + +_REPO_ROOT = _DEV_TOOLS.parent +_LICENSES = _REPO_ROOT / "3rd_party" / "licenses" + +_HEADER = "name,version,revision,url,license,copyright,sourceURL\n" + + +def _write_info(directory: Path, filename: str, row: str) -> None: + (directory / filename).write_text(_HEADER + row + "\n") + + +def test_mapped_component_becomes_purl_component(tmp_path: Path) -> None: + _write_info(tmp_path, "boost-INFO.csv", "Boost C++ Libraries,1.86.0,,http://www.boost.org,BSL-1.0,,") + + sbom, skipped = gs.build_sbom(tmp_path) + + assert skipped == [] + [component] = sbom["components"] + assert component["name"] == "boost" + assert component["version"] == "1.86.0" + assert component["purl"] == "pkg:generic/boost@1.86.0" + + +def test_pytorch_maps_to_pypi_torch(tmp_path: Path) -> None: + _write_info(tmp_path, "pytorch-INFO.csv", "PyTorch,2.7.1,abc123,https://pytorch.org,BSD-3-Clause,,") + + sbom, _ = gs.build_sbom(tmp_path) + + [component] = sbom["components"] + assert component["purl"] == "pkg:pypi/torch@2.7.1" + + +def test_unmapped_component_is_reported_not_emitted(tmp_path: Path) -> None: + _write_info(tmp_path, "scowl-INFO.csv", "SCOWL,7.1,,http://wordlist.aspell.net,Custom,,") + + sbom, skipped = gs.build_sbom(tmp_path) + + assert sbom["components"] == [] + assert skipped == ["SCOWL"] + + +def test_sbom_is_valid_cyclonedx_envelope(tmp_path: Path) -> None: + _write_info(tmp_path, "boost-INFO.csv", "Boost C++ Libraries,1.86.0,,http://www.boost.org,BSL-1.0,,") + + sbom, _ = gs.build_sbom(tmp_path) + + assert sbom["bomFormat"] == "CycloneDX" + assert sbom["specVersion"] == "1.5" + assert sbom["metadata"]["component"]["name"] == "ml-cpp" + # Snyk requires every component to carry a bom-ref and a purl. + for component in sbom["components"]: + assert component["bom-ref"] + assert component["purl"].startswith("pkg:") + + +def test_component_with_empty_version_is_reported(tmp_path: Path) -> None: + _write_info(tmp_path, "boost-INFO.csv", "Boost C++ Libraries,,,http://www.boost.org,BSL-1.0,,") + + sbom, skipped = gs.build_sbom(tmp_path) + + assert sbom["components"] == [] + assert skipped == ["Boost C++ Libraries"] + + +def test_real_license_dir_yields_boost_and_torch() -> None: + sbom, _ = gs.build_sbom(_LICENSES) + + purls = {c["purl"] for c in sbom["components"]} + assert any(p.startswith("pkg:generic/boost@") for p in purls) + assert any(p.startswith("pkg:pypi/torch@") for p in purls) + + +def test_main_writes_json_file(tmp_path: Path) -> None: + out = tmp_path / "sbom.cdx.json" + + rc = gs.main(["--licenses-dir", str(_LICENSES), "--output", str(out)]) + + assert rc == 0 + assert json.loads(out.read_text())["bomFormat"] == "CycloneDX" + + +def test_main_fails_when_nothing_is_scannable(tmp_path: Path) -> None: + empty = tmp_path / "licenses" + empty.mkdir() + + with pytest.raises(SystemExit) as exc: + gs.main(["--licenses-dir", str(empty), "--output", str(tmp_path / "out.json")]) + + assert exc.value.code != 0