[ML] Scan shipped third-party libraries with Snyk in CI

.buildkite/hooks/post-checkout

@@ -48,6 +48,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == ml-cpp* ]]; then
     fi
   fi
 
+  if [[ "$BUILDKITE_STEP_KEY" == "snyk_sbom_scan" ]]; then
+    SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-ml-cpp/snyk/api_token 2>/dev/null || echo "")
+    if [ -n "$SNYK_TOKEN" ]; then
+      export SNYK_TOKEN
+    fi
+  fi
+
   if [[ "$BUILDKITE_STEP_KEY" == "validate_pytorch_allowlist" ]]; then
     HF_TOKEN=$(vault read -field=token secret/ci/elastic-ml-cpp/huggingface/hf_token 2>/dev/null || echo "")
     if [ -n "$HF_TOKEN" ]; then

.buildkite/pipeline.json.py

@@ -125,6 +125,12 @@ def main():
                                                        ".buildkite/pipelines/check_build_regression.yml.sh",
                                                        soft_fail=True))
 
+    # Scan the shipped third-party libraries (Boost, libtorch, ...) with Snyk; the
+    # Snyk GitHub integration only sees the build-image Dockerfiles.
+    pipeline_steps.append(pipeline_steps.generate_step("Upload Snyk scan of shipped third-party libraries",
+                                                       ".buildkite/pipelines/snyk_sbom_scan.yml.sh",
+                                                       soft_fail=True))
+
     # Validate the PyTorch allowlist against HuggingFace models when
     # triggered from the PyTorch edge pipeline.  Runs in a python:3
     # container since the build/test images don't include Python.

.buildkite/pipelines/snyk_sbom_scan.yml.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+# Scan the third-party libraries that ship in the ml-cpp artifacts with Snyk.
+# The Snyk GitHub integration only scans the Dockerfiles (build images), so the
+# source-built libraries that actually ship (Boost, libtorch, ...) are checked
+# here from an SBOM generated from 3rd_party/licenses/*INFO.csv.
+#
+# soft_fail while the baseline is being understood: findings turn the step
+# yellow without failing the build.
+
+cat <<'EOL'
+steps:
+  - label: "Snyk scan of shipped third-party libraries :snyk:"
+    key: "snyk_sbom_scan"
+    command:
+        - ".buildkite/scripts/steps/snyk-sbom-scan.sh"
+    soft_fail: true
+    agents:
+      image: "python:3.11-bookworm"
+    notify:
+      - github_commit_status:
+          context: "Snyk scan of shipped third-party libraries"
+EOL

.buildkite/scripts/steps/snyk-sbom-scan.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+# Generate an SBOM of the shipped third-party libraries and test it with Snyk.
+# SNYK_TOKEN is injected by .buildkite/hooks/post-checkout for this step key.
+
+set -euo pipefail
+
+REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}"
+cd "${REPO_ROOT}"
+
+SNYK_ORG="${SNYK_ORG:-machine-learning-48e}"
+SNYK_SEVERITY_THRESHOLD="${SNYK_SEVERITY_THRESHOLD:-high}"
+SBOM_FILE="${SBOM_FILE:-sbom.cdx.json}"
+
+python3 dev-tools/generate_sbom.py --output "${SBOM_FILE}"
+
+if [ -z "${SNYK_TOKEN:-}" ]; then
+    # Fail loudly rather than reporting a green scan that never ran. The step is
+    # soft_fail, so this shows up as a warning and does not block the build.
+    echo "ERROR: SNYK_TOKEN is not set; the Snyk scan did not run." >&2
+    echo "Expected the post-checkout hook to read it from Vault for step key snyk_sbom_scan." >&2
+    exit 1
+fi
+
+# Install the Snyk CLI and verify the published checksum before running it.
+SNYK_BIN="$(mktemp -d)/snyk"
+curl -fsSL -o "${SNYK_BIN}" https://static.snyk.io/cli/latest/snyk-linux
+EXPECTED_SHA="$(curl -fsSL https://static.snyk.io/cli/latest/snyk-linux.sha256 | awk '{print $1}')"
+echo "${EXPECTED_SHA}  ${SNYK_BIN}" | sha256sum -c -
+chmod +x "${SNYK_BIN}"
+
+# Non-zero exit when vulnerabilities at or above the threshold are found.
+"${SNYK_BIN}" sbom test \
+    --file="${SBOM_FILE}" \
+    --org="${SNYK_ORG}" \
+    --severity-threshold="${SNYK_SEVERITY_THRESHOLD}"

dev-tools/generate_sbom.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+"""Generate a CycloneDX SBOM of the third-party libraries ml-cpp ships.
+
+The Snyk integration on this repo scans the Dockerfiles, i.e. the build images,
+which says nothing about what ends up in the released zip. The libraries that do
+ship (Boost, libtorch, ...) are built from source, so Snyk cannot see them.
+
+This script turns the per-component ``3rd_party/licenses/*INFO.csv`` files (the
+source of truth for shipped versions, also used by dependency_report.sh) into a
+CycloneDX JSON document that ``snyk sbom test`` can analyse.
+
+Only components Snyk can match are emitted. Snyk resolves ``pkg:pypi/...`` through
+its PyPI database and some ``pkg:generic/...`` names through its unmanaged C/C++
+database. A component Snyk does not recognise is silently ignored by the scan, so
+those are reported on stderr instead of being emitted: a clean scan must not be
+mistaken for a component having been checked.
+
+To cover another component, add it to ``_PURL_BY_NAME`` after confirming Snyk
+matches it, e.g. by testing an old, known-vulnerable version.
+
+Usage:
+    generate_sbom.py [--licenses-dir DIR] [--output FILE]
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple
+
+_REPO_ROOT = Path(__file__).resolve().parent.parent
+_DEFAULT_LICENSES_DIR = _REPO_ROOT / "3rd_party" / "licenses"
+
+# INFO.csv component name (lowercased) -> (purl type and name, SBOM component name).
+# Each entry has been confirmed to be matched by Snyk; see the module docstring.
+_PURL_BY_NAME: Dict[str, Tuple[str, str]] = {
+    "boost c++ libraries": ("generic/boost", "boost"),
+    "libxml2": ("generic/libxml2", "libxml2"),
+    # libtorch is built from the PyTorch sources that the PyPI "torch" package ships.
+    "pytorch": ("pypi/torch", "torch"),
+}
+
+
+def _read_info_rows(licenses_dir: Path) -> List[Dict[str, str]]:
+    rows: List[Dict[str, str]] = []
+    for info_file in sorted(licenses_dir.glob("*INFO.csv")):
+        with info_file.open(newline="") as handle:
+            file_rows = list(csv.DictReader(handle))
+        # dependency_report.cmake only uses the last line of each file.
+        if file_rows:
+            rows.append(file_rows[-1])
+    return rows
+
+
+def build_sbom(licenses_dir: Path) -> Tuple[dict, List[str]]:
+    """Return (CycloneDX document, names of components that could not be emitted)."""
+    components: List[dict] = []
+    skipped: List[str] = []
+
+    for row in _read_info_rows(licenses_dir):
+        name = row.get("name", "").strip()
+        version = row.get("version", "").strip()
+        mapping: Optional[Tuple[str, str]] = _PURL_BY_NAME.get(name.lower())
+        if mapping is None or not version:
+            skipped.append(name)
+            continue
+        purl_path, component_name = mapping
+        purl = f"pkg:{purl_path}@{version}"
+        components.append(
+            {
+                "type": "library",
+                "name": component_name,
+                "version": version,
+                "purl": purl,
+                "bom-ref": purl,
+            }
+        )
+
+    sbom = {
+        "bomFormat": "CycloneDX",
+        "specVersion": "1.5",
+        "version": 1,
+        "metadata": {"component": {"type": "application", "name": "ml-cpp", "version": "0"}},
+        "components": components,
+    }
+    return sbom, skipped
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0])
+    parser.add_argument("--licenses-dir", type=Path, default=_DEFAULT_LICENSES_DIR)
+    parser.add_argument("--output", type=Path, default=Path("sbom.cdx.json"))
+    args = parser.parse_args(argv)
+
+    sbom, skipped = build_sbom(args.licenses_dir)
+
+    if skipped:
+        print(
+            "Not scanned (no Snyk-matchable purl mapping): " + ", ".join(skipped),
+            file=sys.stderr,
+        )
+    if not sbom["components"]:
+        sys.exit("error: no scannable components found in " + str(args.licenses_dir))
+
+    args.output.write_text(json.dumps(sbom, indent=2) + "\n")
+    print(
+        "Wrote {} component(s) to {}: {}".format(
+            len(sbom["components"]),
+            args.output,
+            ", ".join(c["purl"] for c in sbom["components"]),
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

dev-tools/unittest/test_generate_sbom.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+"""Pytest tests for dev-tools/generate_sbom.py.
+
+Verifies the CycloneDX SBOM built from 3rd_party/licenses/*INFO.csv only lists
+components that Snyk can match, and that components it cannot match are reported
+rather than silently dropped.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_DEV_TOOLS = Path(__file__).resolve().parents[1]
+if str(_DEV_TOOLS) not in sys.path:
+    sys.path.insert(0, str(_DEV_TOOLS))
+
+import generate_sbom as gs  # noqa: E402
+
+_REPO_ROOT = _DEV_TOOLS.parent
+_LICENSES = _REPO_ROOT / "3rd_party" / "licenses"
+
+_HEADER = "name,version,revision,url,license,copyright,sourceURL\n"
+
+
+def _write_info(directory: Path, filename: str, row: str) -> None:
+    (directory / filename).write_text(_HEADER + row + "\n")
+
+
+def test_mapped_component_becomes_purl_component(tmp_path: Path) -> None:
+    _write_info(tmp_path, "boost-INFO.csv", "Boost C++ Libraries,1.86.0,,http://www.boost.org,BSL-1.0,,")
+
+    sbom, skipped = gs.build_sbom(tmp_path)
+
+    assert skipped == []
+    [component] = sbom["components"]
+    assert component["name"] == "boost"
+    assert component["version"] == "1.86.0"
+    assert component["purl"] == "pkg:generic/boost@1.86.0"
+
+
+def test_pytorch_maps_to_pypi_torch(tmp_path: Path) -> None:
+    _write_info(tmp_path, "pytorch-INFO.csv", "PyTorch,2.7.1,abc123,https://pytorch.org,BSD-3-Clause,,")
+
+    sbom, _ = gs.build_sbom(tmp_path)
+
+    [component] = sbom["components"]
+    assert component["purl"] == "pkg:pypi/torch@2.7.1"
+
+
+def test_unmapped_component_is_reported_not_emitted(tmp_path: Path) -> None:
+    _write_info(tmp_path, "scowl-INFO.csv", "SCOWL,7.1,,http://wordlist.aspell.net,Custom,,")
+
+    sbom, skipped = gs.build_sbom(tmp_path)
+
+    assert sbom["components"] == []
+    assert skipped == ["SCOWL"]
+
+
+def test_sbom_is_valid_cyclonedx_envelope(tmp_path: Path) -> None:
+    _write_info(tmp_path, "boost-INFO.csv", "Boost C++ Libraries,1.86.0,,http://www.boost.org,BSL-1.0,,")
+
+    sbom, _ = gs.build_sbom(tmp_path)
+
+    assert sbom["bomFormat"] == "CycloneDX"
+    assert sbom["specVersion"] == "1.5"
+    assert sbom["metadata"]["component"]["name"] == "ml-cpp"
+    # Snyk requires every component to carry a bom-ref and a purl.
+    for component in sbom["components"]:
+        assert component["bom-ref"]
+        assert component["purl"].startswith("pkg:")
+
+
+def test_component_with_empty_version_is_reported(tmp_path: Path) -> None:
+    _write_info(tmp_path, "boost-INFO.csv", "Boost C++ Libraries,,,http://www.boost.org,BSL-1.0,,")
+
+    sbom, skipped = gs.build_sbom(tmp_path)
+
+    assert sbom["components"] == []
+    assert skipped == ["Boost C++ Libraries"]
+
+
+def test_real_license_dir_yields_boost_and_torch() -> None:
+    sbom, _ = gs.build_sbom(_LICENSES)
+
+    purls = {c["purl"] for c in sbom["components"]}
+    assert any(p.startswith("pkg:generic/boost@") for p in purls)
+    assert any(p.startswith("pkg:pypi/torch@") for p in purls)
+
+
+def test_main_writes_json_file(tmp_path: Path) -> None:
+    out = tmp_path / "sbom.cdx.json"
+
+    rc = gs.main(["--licenses-dir", str(_LICENSES), "--output", str(out)])
+
+    assert rc == 0
+    assert json.loads(out.read_text())["bomFormat"] == "CycloneDX"
+
+
+def test_main_fails_when_nothing_is_scannable(tmp_path: Path) -> None:
+    empty = tmp_path / "licenses"
+    empty.mkdir()
+
+    with pytest.raises(SystemExit) as exc:
+        gs.main(["--licenses-dir", str(empty), "--output", str(tmp_path / "out.json")])
+
+    assert exc.value.code != 0