From d4a235e0c6be5ce4c32e2ed65bc938de0e4a498b Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:58:52 +0800 Subject: [PATCH 01/10] [None][infra] validate latest CBTS coverage DB compatibility Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/scripts/cbts/README.md | 2 + .../cbts/coverage_selection/SELECTION.md | 52 ++-- .../cbts/coverage_selection/artifact.py | 237 ++++++++++++---- .../scripts/test_cbts_coverage_artifact.py | 263 ++++++++++++++++++ 4 files changed, 484 insertions(+), 70 deletions(-) create mode 100644 tests/unittest/scripts/test_cbts_coverage_artifact.py diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 60251c871d65..eb39f3440b65 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -325,6 +325,8 @@ CBTS defers to the existing filter chain when: decorator line), has no usable patch, has unparsable source, or has a closure change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs +- The current PR diff cannot be applied without conflicts to the latest complete coverage DB + revision (or the check cannot be completed) — Tier 2 declines before download - The resolved DB sits more than `--coverage-max-drift` commits from the PR's base commit, on either side, or an unmeasurable distance from it — Tier 2 declines (`coverage_freshness` = `stale` / `unknown`) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 64188f38c506..7ce46c5b9962 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -161,15 +161,18 @@ Jenkins REST lastBuild → newest build number N for b in N .. N-49: (_MAX_PROBE) ranged GET both architecture tarballs → skip b unless both exist GET build_info.txt, parse `commit=` → sha; skip when absent - compare ... → retain ancestors and exact matches - retain distance to the PR base -rank by (distance ascending, build descending) + first complete pair with a known sha → latest coverage DB + compare ... → record topology and absolute distance +fetch PR head, base, and latest DB into a temporary bare repository +apply `git diff ` to the DB revision with `git apply --3way` + clean → continue + conflict / unavailable revision → decline Tier 2 ``` -Requiring the pair prevents the selector from narrowing only one CPU architecture. Ranking is by -**revision, not build number**: a post-merge build can be a re-run of an older commit, so the -highest build number is not necessarily the closest safe revision. Build number only breaks ties. -An exact PR-base match is allowed and wins with distance zero. +Requiring the pair prevents the selector from narrowing only one CPU architecture. Builds are +probed newest first, and the first complete pair with commit metadata is the only DB considered. +The selector does not substitute an older DB merely because it is closer to the PR base or because +the latest DB conflicts with the PR diff. ### 8.2 Measuring the lag (reporting) @@ -181,10 +184,11 @@ Since every candidate revision is a commit that already merged to `main`, it can *behind* the tip: `behind_by` stays 0 and the lag is non-negative. A non-zero `behind_by` would mean the revision is no longer on `main` at all (history rewritten). -There is no local-git path. The CI checkout is `depth: 1, noTags: true` with a single-SHA refspec -(`trtllm_utils.checkoutSpec`), so no candidate revision is ever in the object store; a git -measurement would also answer against whatever ref it was given, and a merely stale ref returns a -*smaller* number rather than an error. +There is no local-git path for measuring lag. The CI checkout is `depth: 1, noTags: true` with a +single-SHA refspec (`trtllm_utils.checkoutSpec`), so the coverage revision is not available there. +The conflict check creates a temporary bare repository, fetches the checked-out PR head locally +and the public main revisions from GitHub, then uses an isolated index. It never changes the CI +checkout or its index. The compare API answers unless the revision has not reached the public mirror yet (404) or the token is missing (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is @@ -195,26 +199,28 @@ The token comes from the `github-cred-trtllm-ci` credential — the one `getGith already uses — bound around the `--print-selection` call in `_cbtsCoverageAudit` and read from `GITHUB_API_TOKEN`. -This number reports overall freshness. It is **not** what the gate decides on. +This number reports overall freshness. It does not select the DB. ### 8.2b Measuring the drift (gating) -The PR base is `merge_base_commit.sha` from `main...`. Each candidate is compared as -`...` and is eligible only when GitHub reports `ahead` (the PR base is ahead of the -DB) or `identical`. `behind`, `diverged`, and unknown relations are rejected before download, so a -coverage DB is never newer than the PR base. +The PR base is `merge_base_commit.sha` from `main...`. The latest DB is compared as +`...`, and drift is `ahead_by + behind_by`: an absolute distance used only by the +freshness gate and telemetry. `ahead`, `behind`, and `identical` describe valid positions on main; +diverged and unknown relations decline Tier 2. -For eligible candidates, drift is their plain ancestor distance to the PR base. The closest one -wins, and `--coverage-max-drift` applies a second fail-closed bound: beyond 30 commits Tier 2 -declines and the PR runs in full. +The latest DB must accept the complete binary PR diff from base to head via a three-way `git apply` +with no conflicts. Conflict, fetch failure, or an unconstructable diff declines Tier 2 before the +large DB artifacts are downloaded. If it passes, `--coverage-max-drift` applies a second fail-closed +bound: beyond 30 commits Tier 2 declines and the PR runs in full. ### 8.3 What happens with the result -`--prepare DIR` does the whole fetch in one call: resolve the PR base, select a complete pair, -stream both tarballs down, unpack their identically named SQLite files separately, and union them -with `compact_db.merge_databases`. It writes the selection JSON beside the merged SQLite as -`cbts_coverage_db.json` and prints `{path, meta}`. Groovy is left with the two things only it can do +`--prepare DIR` does the whole fetch in one call: resolve the PR base, select the latest complete +pair, validate the PR diff, stream both tarballs down, unpack their identically named SQLite files +separately, and union them with `compact_db.merge_databases`. It writes the selection JSON beside +the merged SQLite as `cbts_coverage_db.json` and prints `{path, meta}`. Groovy is left with the two +things only it can do — bind the credential and run `coverage_audit.py` over the result — and any failure anywhere is caught and non-fatal: no prepared DB is returned, Tier 2 never runs, and the PR gets a full run. diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index ad2621d294a1..014b1d2b9abd 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -14,19 +14,19 @@ """Resolve which post-merge CBTS touch DBs to use. Candidates are recent builds of `` with both x86 and SBSA -coverage tarballs. A candidate must have collected a revision at or before the -PR base; the candidate closest to that base wins, with build number only a -tie-break. Revision ordering comes from the forge compare API — the CI checkout -is depth-1, so git cannot answer it — and needs `GITHUB_API_TOKEN`, the anonymous -quota being per-IP and exhausted by shared CI egress. +coverage tarballs. The newest complete pair wins. The PR diff must apply +cleanly to that DB's revision; otherwise Tier 2 declines rather than choosing +an older DB. Revision metadata comes from the forge compare API and needs +`GITHUB_API_TOKEN`, the anonymous quota being per-IP and exhausted by shared CI +egress. The selected architecture DBs are merged locally through the compact coverage schema, producing the one DB consumed by `main.py`. Two entry points. `--print-selection` prints `{urls, build, commit, lag, -base_commit, drift, drift_status}` and stops. `--prepare DIR` goes on to -download, unpack, and merge the winner, drop that JSON beside it, and print -`{path, meta}` — the two paths `main.py` needs. +base_commit, drift, drift_status, patch_apply_status}` and stops. `--prepare +DIR` goes on to download, unpack, and merge the winner, drop that JSON beside +it, and print `{path, meta}` — the two paths `main.py` needs. """ from __future__ import annotations @@ -36,6 +36,7 @@ import os import shutil import sqlite3 +import subprocess import sys import tarfile import tempfile @@ -43,7 +44,7 @@ import urllib.request from functools import lru_cache from pathlib import Path -from typing import Optional +from typing import Literal, Optional sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "coverage_utils")) @@ -68,6 +69,7 @@ _URM = "https://urm.nvidia.com/artifactory" _GITHUB_COMPARE = "https://api.github.com/repos/NVIDIA/TensorRT-LLM/compare" +_GITHUB_GIT_REPO = "https://github.com/NVIDIA/TensorRT-LLM.git" _JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" # Cover the 30-commit freshness window plus missing or unsuccessful builds. _MAX_PROBE = 50 @@ -78,6 +80,8 @@ # Tarball download attempts. _RETRIES = 3 +_PatchApplyStatus = Literal["clean", "conflict", "unknown"] + def _get(url: str, headers: Optional[dict] = None) -> tuple[Optional[int], Optional[bytes]]: req = urllib.request.Request(url, headers=headers or {}) @@ -204,18 +208,159 @@ def drift(db_commit: str, base_commit: str) -> tuple[Optional[int], str]: return None, "unknown" +def _run_git( + args: list[str], + *, + cwd: Path, + input_data: Optional[bytes] = None, + env: Optional[dict[str, str]] = None, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + ["git", *args], + cwd=cwd, + input=input_data, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + ) + + +def _patch_apply_status( + pr_base_commit: str, + pr_head: str, + db_commit: str, + repo_root: Path = Path("."), + upstream_url: str = _GITHUB_GIT_REPO, +) -> _PatchApplyStatus: + """Return clean/conflict/unknown for applying the PR diff to the DB revision.""" + repo_source = repo_root.resolve() + checked_out_head = _run_git(["rev-parse", "HEAD"], cwd=repo_source) + if ( + checked_out_head.returncode != 0 + or checked_out_head.stdout.decode("ascii", "replace").strip() != pr_head + ): + print(f"[artifact] checked-out revision is not PR head {pr_head[:10]}", file=sys.stderr) + return "unknown" + with tempfile.TemporaryDirectory(prefix="cbts_patch_check_") as temp_dir: + temp = Path(temp_dir) + git_dir = temp / "repo.git" + initialized = _run_git(["init", "--bare", str(git_dir)], cwd=temp) + if initialized.returncode != 0: + print("[artifact] could not initialize the patch-check repository", file=sys.stderr) + return "unknown" + + # Fetching into a temporary bare repository leaves the shallow CI checkout and its + # index untouched. The checked-out PR head is available locally; both main revisions + # are fetched from the public upstream repository. + fetched_head = _run_git( + [ + "--git-dir", + str(git_dir), + "fetch", + "--no-tags", + "--depth=1", + str(repo_source), + "HEAD", + ], + cwd=temp, + ) + if fetched_head.returncode != 0: + print( + f"[artifact] could not load PR head {pr_head[:10]} for patch check", + file=sys.stderr, + ) + return "unknown" + + revisions = list(dict.fromkeys((pr_base_commit, db_commit))) + fetched_revisions = _run_git( + [ + "--git-dir", + str(git_dir), + "fetch", + "--no-tags", + "--depth=1", + upstream_url, + *revisions, + ], + cwd=temp, + ) + if fetched_revisions.returncode != 0: + print( + "[artifact] could not load the base/coverage revisions for patch check", + file=sys.stderr, + ) + return "unknown" + + diff = _run_git( + [ + "--git-dir", + str(git_dir), + "diff", + "--binary", + "--full-index", + pr_base_commit, + pr_head, + ], + cwd=temp, + ) + if diff.returncode != 0: + print("[artifact] could not generate the PR diff for patch check", file=sys.stderr) + return "unknown" + + index = temp / "index" + git_env = os.environ.copy() + git_env["GIT_INDEX_FILE"] = str(index) + read_tree = _run_git( + ["--git-dir", str(git_dir), "read-tree", db_commit], cwd=temp, env=git_env + ) + if read_tree.returncode != 0: + print( + "[artifact] could not read the coverage revision for patch check", + file=sys.stderr, + ) + return "unknown" + + applied = _run_git( + [ + "--git-dir", + str(git_dir), + "apply", + "--cached", + "--3way", + "--whitespace=nowarn", + "-", + ], + cwd=temp, + input_data=diff.stdout, + env=git_env, + ) + return "clean" if applied.returncode == 0 else "conflict" + + +def _selection_accepts_pr_diff(sel: dict, pr_base_commit: str, pr_head: str) -> bool: + sel["patch_apply_status"] = _patch_apply_status(pr_base_commit, pr_head, sel["commit"]) + if sel["patch_apply_status"] == "clean": + return True + print( + f"[artifact] coverage tier declined: PR diff does not apply cleanly to " + f"latest DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})", + file=sys.stderr, + ) + return False + + def select_tarball( pr_base_commit: str, artifact_base: str = ARTIFACT_BASE, jenkins_base: str = _JENKINS_BASE, max_probe: int = _MAX_PROBE, ) -> Optional[dict]: - """Closest complete architecture pair collected at or before `pr_base_commit`.""" + """Newest complete architecture pair with measurable PR-base topology.""" build = latest_build_number(jenkins_base) if build is None: print("[artifact] could not resolve latest build number", file=sys.stderr) return None - candidates = [] for b in range(build, max(0, build - max_probe), -1): urls = tarball_urls(b, artifact_base) if not all(_exists(url) for url in urls): @@ -227,42 +372,35 @@ def select_tarball( distance, status = drift(commit, pr_base_commit) if distance is None: print( - f"[artifact] build {b}: ordering against the PR base unknown; skipped", + f"[artifact] latest complete build {b}: topology against the PR base unknown", file=sys.stderr, ) - continue - if status not in ("ahead", "identical"): + return None + if status not in ("ahead", "behind", "identical"): print( - f"[artifact] build {b}: relation to the PR base is {status or 'unknown'}; skipped", + f"[artifact] latest complete build {b}: relation to the PR base is " + f"{status or 'unknown'}", file=sys.stderr, ) - continue - candidates.append( - { - "url": urls[0], - "urls": urls, - "build": b, - "commit": commit, - "base_commit": pr_base_commit, - "drift": distance, - "drift_status": status, - } - ) - if not candidates: - print( - f"[artifact] no complete x86/SBSA coverage pair at or before the PR base " - f"in the last {max_probe} builds", - file=sys.stderr, - ) - return None - best = min(candidates, key=lambda c: (c["drift"], -c["build"])) - best["lag"] = compare_distance(best["commit"]) - if best["lag"] is None: - print( - f"[artifact] build {best['build']}: lag behind main unknown", - file=sys.stderr, - ) - return best + return None + selected = { + "url": urls[0], + "urls": urls, + "build": b, + "commit": commit, + "base_commit": pr_base_commit, + "drift": distance, + "drift_status": status, + "lag": compare_distance(commit), + } + if selected["lag"] is None: + print(f"[artifact] build {b}: lag behind main unknown", file=sys.stderr) + return selected + print( + f"[artifact] no complete x86/SBSA coverage pair in the last {max_probe} builds", + file=sys.stderr, + ) + return None def measure_drift(sel: dict, pr_head: Optional[str]) -> dict: @@ -336,7 +474,7 @@ def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: Paths are relative to the caller's cwd, matching the Groovy caller's `cd ${LLM_ROOT}`. """ if not pr_head: - print("[artifact] PR head is required to select an ancestor coverage DB", file=sys.stderr) + print("[artifact] PR head is required to select a coverage DB", file=sys.stderr) return None pr_base_commit = merge_base(pr_head) if not pr_base_commit: @@ -345,6 +483,8 @@ def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: sel = select_tarball(pr_base_commit) if sel is None: return None + if not _selection_accepts_pr_diff(sel, pr_base_commit, pr_head): + return None print(describe(sel), file=sys.stderr) dest = Path(dest_dir) @@ -383,7 +523,8 @@ def main(argv: Optional[list[str]] = None) -> int: ap.add_argument( "--print-selection", action="store_true", - help="resolve and print {urls, build, commit, lag, base_commit, drift, drift_status} as JSON", + help="resolve and print {urls, build, commit, lag, base_commit, drift, " + "drift_status, patch_apply_status} as JSON", ) ap.add_argument( "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" @@ -398,8 +539,8 @@ def main(argv: Optional[list[str]] = None) -> int: ap.add_argument( "--pr-head", default=None, - help="required PR head revision; its merge base is the inclusive upper bound " - "for eligible coverage revisions", + help="required PR head revision; its merge base measures DB drift and defines " + "the diff checked against the latest coverage revision", ) args = ap.parse_args(argv) @@ -433,12 +574,14 @@ def main(argv: Optional[list[str]] = None) -> int: "lag": compare_distance(commit) if commit else None, } measure_drift(best, args.pr_head) - if best["drift_status"] not in ("ahead", "identical"): + if best["drift"] is None or best["drift_status"] not in ("ahead", "behind", "identical"): return 1 else: best = select_tarball(pr_base_commit) if best is None: return 1 + if not _selection_accepts_pr_diff(best, pr_base_commit, args.pr_head): + return 1 print(json.dumps(best)) return 0 diff --git a/tests/unittest/scripts/test_cbts_coverage_artifact.py b/tests/unittest/scripts/test_cbts_coverage_artifact.py new file mode 100644 index 000000000000..d2fb8a5dc9aa --- /dev/null +++ b/tests/unittest/scripts/test_cbts_coverage_artifact.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for CBTS coverage artifact selection and architecture DB merging.""" + +from __future__ import annotations + +import ast +import importlib.util +import json +import shutil +import sqlite3 +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import ModuleType +from unittest import mock + +_ROOT = Path(__file__).resolve().parents[3] +_CBTS = _ROOT / "jenkins/scripts/cbts" +_ARTIFACT_PATH = _CBTS / "coverage_selection/artifact.py" +_MAIN_PATH = _CBTS / "main.py" +sys.path.insert(0, str(_CBTS / "coverage_utils")) + +from compact_db import write_leaf_database # noqa: E402 + + +def _load_artifact() -> ModuleType: + spec = importlib.util.spec_from_file_location("cbts_coverage_artifact", _ARTIFACT_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot load {_ARTIFACT_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +artifact = _load_artifact() + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + +class CoverageArtifactTest(unittest.TestCase): + def test_selects_latest_complete_pair_even_when_an_older_db_is_closer(self) -> None: + commits = {104: "newer", 102: "older-three", 101: "older-one"} + + def exists(url: str) -> bool: + if "/103/" in url: + return url.endswith("cbts_pystart_report_x86_64.tar.gz") + return "/100/" not in url + + relations = { + "newer": (8, "behind"), + "older-three": (3, "ahead"), + "older-one": (1, "ahead"), + } + with ( + mock.patch.object(artifact, "latest_build_number", return_value=104), + mock.patch.object(artifact, "_exists", side_effect=exists), + mock.patch.object( + artifact, "build_commit", side_effect=lambda build, _base: commits[build] + ) as build_commit, + mock.patch.object( + artifact, "drift", side_effect=lambda commit, _base: relations[commit] + ), + mock.patch.object(artifact, "compare_distance", return_value=7) as lag, + ): + selected = artifact.select_tarball( + "pr-base", artifact_base="coverage", jenkins_base="jenkins", max_probe=5 + ) + + self.assertIsNotNone(selected) + assert selected is not None + self.assertEqual(selected["build"], 104) + self.assertEqual(selected["commit"], "newer") + self.assertEqual(selected["drift"], 8) + self.assertEqual(selected["drift_status"], "behind") + self.assertEqual( + [url.rsplit("/", 1)[-1] for url in selected["urls"]], + list(artifact.ARCH_TARBALL_NAMES), + ) + build_commit.assert_called_once_with(104, "coverage") + lag.assert_called_once_with("newer") + + def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo = Path(temp_dir) + _git(repo, "init") + _git(repo, "config", "user.email", "cbts@example.com") + _git(repo, "config", "user.name", "CBTS Test") + source = repo / "source.py" + source.write_text("first\nbase\nlast\n") + _git(repo, "add", "source.py") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD") + + _git(repo, "switch", "-c", "pr") + source.write_text("first\npr\nlast\n") + _git(repo, "commit", "-am", "pr") + head = _git(repo, "rev-parse", "HEAD") + + _git(repo, "switch", "-c", "db-clean", base) + (repo / "other.py").write_text("coverage revision\n") + _git(repo, "add", "other.py") + _git(repo, "commit", "-m", "non-conflicting db") + clean_db = _git(repo, "rev-parse", "HEAD") + + _git(repo, "switch", "-c", "db-conflict", base) + source.write_text("first\ndb\nlast\n") + _git(repo, "commit", "-am", "conflicting db") + conflicting_db = _git(repo, "rev-parse", "HEAD") + _git(repo, "switch", "pr") + + self.assertEqual( + artifact._patch_apply_status(base, head, clean_db, repo, str(repo)), "clean" + ) + self.assertEqual( + artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)), + "conflict", + ) + + def test_accepts_artifact_collected_at_pr_base(self) -> None: + with ( + mock.patch.object(artifact, "latest_build_number", return_value=7), + mock.patch.object(artifact, "_exists", return_value=True), + mock.patch.object(artifact, "build_commit", return_value="pr-base"), + mock.patch.object(artifact, "drift", return_value=(0, "identical")), + mock.patch.object(artifact, "compare_distance", return_value=4), + ): + selected = artifact.select_tarball("pr-base", max_probe=1) + + self.assertIsNotNone(selected) + assert selected is not None + self.assertEqual(selected["drift"], 0) + self.assertEqual(selected["drift_status"], "identical") + + def test_prepare_merges_x86_and_sbsa_databases(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + x86_db = root / "x86.sqlite" + sbsa_db = root / "sbsa.sqlite" + output_dir = root / "prepared" + write_leaf_database( + x86_db, + stage="A10-PyTorch-1", + process_uid="A10-PyTorch-1/coordinator", + touches={ + "A10-PyTorch-1/test_x86.py::test_one": { + ("/workspace/tensorrt_llm/x86.py", "run") + } + }, + outcomes={"A10-PyTorch-1/test_x86.py::test_one": "passed"}, + expected_workers={"A10-PyTorch-1/test_x86.py::test_one": 0}, + ) + write_leaf_database( + sbsa_db, + stage="GH200-PyTorch-1", + process_uid="GH200-PyTorch-1/coordinator", + touches={ + "GH200-PyTorch-1/test_sbsa.py::test_two": { + ("/workspace/tensorrt_llm/sbsa.py", "run") + } + }, + outcomes={"GH200-PyTorch-1/test_sbsa.py::test_two": "passed"}, + expected_workers={"GH200-PyTorch-1/test_sbsa.py::test_two": 0}, + ) + urls = artifact.tarball_urls(42) + selection = { + "url": urls[0], + "urls": urls, + "build": 42, + "commit": "coverage-commit", + "lag": 5, + "base_commit": "pr-base", + "drift": 2, + "drift_status": "ahead", + } + + def download(url: str, destination: Path) -> Path: + return destination / url.rsplit("/", 1)[-1] + + def extract(tarball: Path, destination: Path) -> bool: + source = x86_db if "x86_64" in tarball.name else sbsa_db + shutil.copyfile(source, destination / artifact.DB_NAME) + return True + + with ( + mock.patch.object(artifact, "merge_base", return_value="pr-base"), + mock.patch.object(artifact, "select_tarball", return_value=selection) as select, + mock.patch.object(artifact, "_patch_apply_status", return_value="clean") as apply, + mock.patch.object(artifact, "download", side_effect=download), + mock.patch.object(artifact, "extract", side_effect=extract), + ): + ready = artifact.prepare(str(output_dir), "pr-head") + + self.assertIsNotNone(ready) + assert ready is not None + select.assert_called_once_with("pr-base") + apply.assert_called_once_with("pr-base", "pr-head", "coverage-commit") + connection = sqlite3.connect(ready["path"]) + try: + tests = { + row[0] for row in connection.execute("SELECT DISTINCT test FROM touch_rows") + } + finally: + connection.close() + self.assertEqual( + tests, + { + "A10-PyTorch-1/test_x86.py::test_one", + "GH200-PyTorch-1/test_sbsa.py::test_two", + }, + ) + self.assertEqual(json.loads(Path(ready["meta"]).read_text()), selection) + + def test_prepare_declines_before_download_when_latest_db_conflicts(self) -> None: + selection = {"commit": "coverage-commit"} + with ( + mock.patch.object(artifact, "merge_base", return_value="pr-base"), + mock.patch.object(artifact, "select_tarball", return_value=selection), + mock.patch.object(artifact, "_patch_apply_status", return_value="conflict"), + mock.patch.object(artifact, "download") as download, + ): + ready = artifact.prepare("unused", "pr-head") + + self.assertIsNone(ready) + download.assert_not_called() + + def test_freshness_default_is_thirty_commits(self) -> None: + tree = ast.parse(_MAIN_PATH.read_text()) + defaults = { + target.id: node.value.value + for node in tree.body + if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant) + for target in node.targets + if isinstance(target, ast.Name) + } + self.assertEqual(defaults["DEFAULT_COVERAGE_MAX_DRIFT"], 30) + + +if __name__ == "__main__": + unittest.main() From 4dda4b16aac644a07b12abfdff275dca5abf2aa4 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:00:21 +0800 Subject: [PATCH 02/10] [TRTLLM-15885][infra] address CBTS compatibility review Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/SELECTION.md | 4 +- .../cbts/coverage_selection/artifact.py | 44 ++++++++++++------- .../scripts/test_cbts_coverage_artifact.py | 42 +++++++++++------- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 7ce46c5b9962..bb512cedbc1c 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -237,10 +237,10 @@ All of it lands in the decision and in OpenSearch: |---|---|---| | `coverage_db_build` | `l_coverage_db_build` | 0 when no DB was consulted | | `coverage_db_commit` | `s_coverage_db_commit` | | -| `coverage_db_lag` | `l_coverage_db_lag` | ranking / overall freshness; `null` / `-1` when unmeasurable | +| `coverage_db_lag` | `l_coverage_db_lag` | overall freshness only; `null` / `-1` when unmeasurable | | `coverage_db_base_commit` | `s_coverage_db_base_commit` | the PR's merge base | | `coverage_db_drift` | `l_coverage_db_drift` | **the gated number**; `null` / `-1` when unmeasurable | -| `coverage_db_drift_status` | `s_coverage_db_drift_status` | `ahead` / `identical` for every selected DB | +| `coverage_db_drift_status` | `s_coverage_db_drift_status` | `ahead` / `behind` / `identical` for every selected DB | | `coverage_freshness` | `s_coverage_freshness` | `ok` / `stale` / `unknown`, empty when no DB | so the decline rate is queryable per verdict rather than only readable in `s_reason`. diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index 014b1d2b9abd..b9fa413458ac 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -77,6 +77,8 @@ _TIMEOUT = 15 # Socket timeout for the tarball itself, which runs to hundreds of MB. _DOWNLOAD_TIMEOUT = 300 +# Timeout for local Git operations and the small fetches used by the patch check. +_GIT_TIMEOUT = 120 # Tarball download attempts. _RETRIES = 3 @@ -214,16 +216,23 @@ def _run_git( cwd: Path, input_data: Optional[bytes] = None, env: Optional[dict[str, str]] = None, -) -> subprocess.CompletedProcess[bytes]: - return subprocess.run( - ["git", *args], - cwd=cwd, - input=input_data, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - check=False, - ) + timeout: int = _GIT_TIMEOUT, +) -> Optional[subprocess.CompletedProcess[bytes]]: + try: + return subprocess.run( + ["git", *args], + cwd=cwd, + input=input_data, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + command = args[0] if args else "command" + print(f"[artifact] git {command} timed out after {timeout}s", file=sys.stderr) + return None def _patch_apply_status( @@ -237,7 +246,8 @@ def _patch_apply_status( repo_source = repo_root.resolve() checked_out_head = _run_git(["rev-parse", "HEAD"], cwd=repo_source) if ( - checked_out_head.returncode != 0 + checked_out_head is None + or checked_out_head.returncode != 0 or checked_out_head.stdout.decode("ascii", "replace").strip() != pr_head ): print(f"[artifact] checked-out revision is not PR head {pr_head[:10]}", file=sys.stderr) @@ -246,7 +256,7 @@ def _patch_apply_status( temp = Path(temp_dir) git_dir = temp / "repo.git" initialized = _run_git(["init", "--bare", str(git_dir)], cwd=temp) - if initialized.returncode != 0: + if initialized is None or initialized.returncode != 0: print("[artifact] could not initialize the patch-check repository", file=sys.stderr) return "unknown" @@ -265,7 +275,7 @@ def _patch_apply_status( ], cwd=temp, ) - if fetched_head.returncode != 0: + if fetched_head is None or fetched_head.returncode != 0: print( f"[artifact] could not load PR head {pr_head[:10]} for patch check", file=sys.stderr, @@ -285,7 +295,7 @@ def _patch_apply_status( ], cwd=temp, ) - if fetched_revisions.returncode != 0: + if fetched_revisions is None or fetched_revisions.returncode != 0: print( "[artifact] could not load the base/coverage revisions for patch check", file=sys.stderr, @@ -304,7 +314,7 @@ def _patch_apply_status( ], cwd=temp, ) - if diff.returncode != 0: + if diff is None or diff.returncode != 0: print("[artifact] could not generate the PR diff for patch check", file=sys.stderr) return "unknown" @@ -314,7 +324,7 @@ def _patch_apply_status( read_tree = _run_git( ["--git-dir", str(git_dir), "read-tree", db_commit], cwd=temp, env=git_env ) - if read_tree.returncode != 0: + if read_tree is None or read_tree.returncode != 0: print( "[artifact] could not read the coverage revision for patch check", file=sys.stderr, @@ -335,6 +345,8 @@ def _patch_apply_status( input_data=diff.stdout, env=git_env, ) + if applied is None: + return "unknown" return "clean" if applied.returncode == 0 else "conflict" diff --git a/tests/unittest/scripts/test_cbts_coverage_artifact.py b/tests/unittest/scripts/test_cbts_coverage_artifact.py index d2fb8a5dc9aa..98f2405099dd 100644 --- a/tests/unittest/scripts/test_cbts_coverage_artifact.py +++ b/tests/unittest/scripts/test_cbts_coverage_artifact.py @@ -115,22 +115,22 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: _git(repo, "commit", "-m", "base") base = _git(repo, "rev-parse", "HEAD") - _git(repo, "switch", "-c", "pr") + _git(repo, "checkout", "-b", "pr") source.write_text("first\npr\nlast\n") _git(repo, "commit", "-am", "pr") head = _git(repo, "rev-parse", "HEAD") - _git(repo, "switch", "-c", "db-clean", base) + _git(repo, "checkout", "-b", "db-clean", base) (repo / "other.py").write_text("coverage revision\n") _git(repo, "add", "other.py") _git(repo, "commit", "-m", "non-conflicting db") clean_db = _git(repo, "rev-parse", "HEAD") - _git(repo, "switch", "-c", "db-conflict", base) + _git(repo, "checkout", "-b", "db-conflict", base) source.write_text("first\ndb\nlast\n") _git(repo, "commit", "-am", "conflicting db") conflicting_db = _git(repo, "rev-parse", "HEAD") - _git(repo, "switch", "pr") + _git(repo, "checkout", "pr") self.assertEqual( artifact._patch_apply_status(base, head, clean_db, repo, str(repo)), "clean" @@ -234,18 +234,30 @@ def extract(tarball: Path, destination: Path) -> bool: ) self.assertEqual(json.loads(Path(ready["meta"]).read_text()), selection) - def test_prepare_declines_before_download_when_latest_db_conflicts(self) -> None: - selection = {"commit": "coverage-commit"} - with ( - mock.patch.object(artifact, "merge_base", return_value="pr-base"), - mock.patch.object(artifact, "select_tarball", return_value=selection), - mock.patch.object(artifact, "_patch_apply_status", return_value="conflict"), - mock.patch.object(artifact, "download") as download, + def test_git_timeout_returns_unknown_result(self) -> None: + with mock.patch.object( + artifact.subprocess, + "run", + side_effect=subprocess.TimeoutExpired(["git", "fetch"], 120), ): - ready = artifact.prepare("unused", "pr-head") - - self.assertIsNone(ready) - download.assert_not_called() + result = artifact._run_git(["fetch"], cwd=Path(".")) + + self.assertIsNone(result) + + def test_prepare_declines_before_download_when_latest_db_is_not_usable(self) -> None: + for patch_status in ("conflict", "unknown"): + with self.subTest(patch_status=patch_status): + selection = {"commit": "coverage-commit"} + with ( + mock.patch.object(artifact, "merge_base", return_value="pr-base"), + mock.patch.object(artifact, "select_tarball", return_value=selection), + mock.patch.object(artifact, "_patch_apply_status", return_value=patch_status), + mock.patch.object(artifact, "download") as download, + ): + ready = artifact.prepare("unused", "pr-head") + + self.assertIsNone(ready) + download.assert_not_called() def test_freshness_default_is_thirty_commits(self) -> None: tree = ast.parse(_MAIN_PATH.read_text()) From c261349f55c5928e42eca79508de8437375244d9 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:15:22 +0800 Subject: [PATCH 03/10] [TRTLLM-15885][test] focus coverage artifact checks Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../scripts/test_cbts_coverage_artifact.py | 79 ------------------- 1 file changed, 79 deletions(-) diff --git a/tests/unittest/scripts/test_cbts_coverage_artifact.py b/tests/unittest/scripts/test_cbts_coverage_artifact.py index 98f2405099dd..0f977150d285 100644 --- a/tests/unittest/scripts/test_cbts_coverage_artifact.py +++ b/tests/unittest/scripts/test_cbts_coverage_artifact.py @@ -17,7 +17,6 @@ from __future__ import annotations -import ast import importlib.util import json import shutil @@ -33,7 +32,6 @@ _ROOT = Path(__file__).resolve().parents[3] _CBTS = _ROOT / "jenkins/scripts/cbts" _ARTIFACT_PATH = _CBTS / "coverage_selection/artifact.py" -_MAIN_PATH = _CBTS / "main.py" sys.path.insert(0, str(_CBTS / "coverage_utils")) from compact_db import write_leaf_database # noqa: E402 @@ -62,47 +60,6 @@ def _git(repo: Path, *args: str) -> str: class CoverageArtifactTest(unittest.TestCase): - def test_selects_latest_complete_pair_even_when_an_older_db_is_closer(self) -> None: - commits = {104: "newer", 102: "older-three", 101: "older-one"} - - def exists(url: str) -> bool: - if "/103/" in url: - return url.endswith("cbts_pystart_report_x86_64.tar.gz") - return "/100/" not in url - - relations = { - "newer": (8, "behind"), - "older-three": (3, "ahead"), - "older-one": (1, "ahead"), - } - with ( - mock.patch.object(artifact, "latest_build_number", return_value=104), - mock.patch.object(artifact, "_exists", side_effect=exists), - mock.patch.object( - artifact, "build_commit", side_effect=lambda build, _base: commits[build] - ) as build_commit, - mock.patch.object( - artifact, "drift", side_effect=lambda commit, _base: relations[commit] - ), - mock.patch.object(artifact, "compare_distance", return_value=7) as lag, - ): - selected = artifact.select_tarball( - "pr-base", artifact_base="coverage", jenkins_base="jenkins", max_probe=5 - ) - - self.assertIsNotNone(selected) - assert selected is not None - self.assertEqual(selected["build"], 104) - self.assertEqual(selected["commit"], "newer") - self.assertEqual(selected["drift"], 8) - self.assertEqual(selected["drift_status"], "behind") - self.assertEqual( - [url.rsplit("/", 1)[-1] for url in selected["urls"]], - list(artifact.ARCH_TARBALL_NAMES), - ) - build_commit.assert_called_once_with(104, "coverage") - lag.assert_called_once_with("newer") - def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: repo = Path(temp_dir) @@ -234,42 +191,6 @@ def extract(tarball: Path, destination: Path) -> bool: ) self.assertEqual(json.loads(Path(ready["meta"]).read_text()), selection) - def test_git_timeout_returns_unknown_result(self) -> None: - with mock.patch.object( - artifact.subprocess, - "run", - side_effect=subprocess.TimeoutExpired(["git", "fetch"], 120), - ): - result = artifact._run_git(["fetch"], cwd=Path(".")) - - self.assertIsNone(result) - - def test_prepare_declines_before_download_when_latest_db_is_not_usable(self) -> None: - for patch_status in ("conflict", "unknown"): - with self.subTest(patch_status=patch_status): - selection = {"commit": "coverage-commit"} - with ( - mock.patch.object(artifact, "merge_base", return_value="pr-base"), - mock.patch.object(artifact, "select_tarball", return_value=selection), - mock.patch.object(artifact, "_patch_apply_status", return_value=patch_status), - mock.patch.object(artifact, "download") as download, - ): - ready = artifact.prepare("unused", "pr-head") - - self.assertIsNone(ready) - download.assert_not_called() - - def test_freshness_default_is_thirty_commits(self) -> None: - tree = ast.parse(_MAIN_PATH.read_text()) - defaults = { - target.id: node.value.value - for node in tree.body - if isinstance(node, ast.Assign) and isinstance(node.value, ast.Constant) - for target in node.targets - if isinstance(target, ast.Name) - } - self.assertEqual(defaults["DEFAULT_COVERAGE_MAX_DRIFT"], 30) - if __name__ == "__main__": unittest.main() From a4810dcda161950371b11d81fa4343107f65aaed Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 10 Sep 2026 13:54:51 +0800 Subject: [PATCH 04/10] [TRTLLM-15885][infra] address coverage DB review Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 37 ++- jenkins/scripts/cbts/README.md | 6 +- .../cbts/coverage_selection/SELECTION.md | 38 +-- .../cbts/coverage_selection/artifact.py | 247 ++++++++++++------ .../scripts/test_cbts_coverage_artifact.py | 32 ++- 5 files changed, 246 insertions(+), 114 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index c8670231c391..b5d593f269b4 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -963,6 +963,12 @@ def getCbtsResult(pipeline, testFilter, globalVars) // Download the touch DB only for PRs in the coverage-tier pilot. def coverageDb = _cbtsCoverageAudit(pipeline) + def coveragePatchInput = null + if (coverageDb?.patch_input) { + coveragePatchInput = new groovy.json.JsonSlurper().parseText( + readFile("${LLM_ROOT}/${coverageDb.patch_input}")) + changedFiles = (coveragePatchInput.changed_files ?: []).unique() + } // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( @@ -975,10 +981,13 @@ def getCbtsResult(pipeline, testFilter, globalVars) } def diffs = [:] if (filesNeedingDiff) { - def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] - def fileChanges = githubPrApiUrl != null - ? getGithubMRChangedFile(pipeline, githubPrApiUrl, "getFileChanges") - : getGitlabMRChangedFile(pipeline, "getFileChanges") + def fileChanges = coveragePatchInput?.diffs + if (fileChanges == null) { + def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] + fileChanges = githubPrApiUrl != null + ? getGithubMRChangedFile(pipeline, githubPrApiUrl, "getFileChanges") + : getGitlabMRChangedFile(pipeline, "getFileChanges") + } diffs = filesNeedingDiff.collectEntries { filePath -> // Null (patch omitted for binary / rename / too-large diffs) coerces to empty. [(filePath): fileChanges[filePath] ?: ""] @@ -1062,7 +1071,7 @@ def _cbtsMultiGpuLabelGateOpen(pipeline, globalVars) } // Check pilot eligibility, then fetch and audit the touch DB; artifact.py's -// {path, meta} verbatim, or null on failure. +// prepared paths and compatibility-diff metadata verbatim, or null on failure. def _cbtsCoverageAudit(pipeline) { try { @@ -1073,7 +1082,13 @@ def _cbtsCoverageAudit(pipeline) def readyJson = "" def prAuthor = "" def pilotEligible = false - withCredentials([usernamePassword(credentialsId: 'github-cred-trtllm-ci', usernameVariable: 'NOT_USED_YET', passwordVariable: 'GITHUB_API_TOKEN')]) { + withCredentials([ + usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN'), + string(credentialsId: 'default-llm-repo', variable: 'CBTS_COVERAGE_GIT_REPO'), + ]) { prAuthor = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_pilot.py", returnStdout: true, @@ -1097,6 +1112,16 @@ def _cbtsCoverageAudit(pipeline) return null } def ready = new groovy.json.JsonSlurper().parseText(readyJson) + if (ready.patch_diff_kind == "cumulative") { + pipeline.echo("CBTS audit: Tier 2 will use cumulative diff " + + "${ready.patch_diff_base_commit.take(10)}.." + + "${ready.patch_diff_head_commit.take(10)}, not the GitHub PR diff " + + "${ready.pr_base_commit.take(10)}.." + + "${ready.patch_diff_head_commit.take(10)}") + } else { + pipeline.echo("CBTS audit: coverage compatibility used GitHub PR diff " + + "${ready.patch_diff_base_commit.take(10)}..${ready.patch_diff_head_commit.take(10)}") + } sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}" return ready } catch (InterruptedException e) { diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index eb39f3440b65..4c0948716801 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -325,8 +325,10 @@ CBTS defers to the existing filter chain when: decorator line), has no usable patch, has unparsable source, or has a closure change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs -- The current PR diff cannot be applied without conflicts to the latest complete coverage DB - revision (or the check cannot be completed) — Tier 2 declines before download +- Neither the current PR diff nor its permitted older-DB cumulative fallback can be applied to the + latest complete coverage DB revision (or the check cannot be completed) — Tier 2 declines before + download. When that fallback is used successfully, Tier 2 uses its DB-to-PR-head changed files + and diffs, and the audit log warns that this input can differ from the diff shown on the PR - The resolved DB sits more than `--coverage-max-drift` commits from the PR's base commit, on either side, or an unmeasurable distance from it — Tier 2 declines (`coverage_freshness` = `stale` / `unknown`) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index bb512cedbc1c..c1853903fc9b 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -163,10 +163,12 @@ for b in N .. N-49: (_MAX_PROBE) GET build_info.txt, parse `commit=` → sha; skip when absent first complete pair with a known sha → latest coverage DB compare ... → record topology and absolute distance -fetch PR head, base, and latest DB into a temporary bare repository -apply `git diff ` to the DB revision with `git apply --3way` +fetch PR head locally; fetch base and latest DB from the normal CI Git mirror +create a squashed PR commit with `commit-tree`, parented at the PR base +cherry-pick it onto the DB revision clean → continue - conflict / unavailable revision → decline Tier 2 + conflict and DB is older than PR base → retry cumulative DB-to-head commit + other conflict / unavailable revision → decline Tier 2 ``` Requiring the pair prevents the selector from narrowing only one CPU architecture. Builds are @@ -186,9 +188,10 @@ mean the revision is no longer on `main` at all (history rewritten). There is no local-git path for measuring lag. The CI checkout is `depth: 1, noTags: true` with a single-SHA refspec (`trtllm_utils.checkoutSpec`), so the coverage revision is not available there. -The conflict check creates a temporary bare repository, fetches the checked-out PR head locally -and the public main revisions from GitHub, then uses an isolated index. It never changes the CI -checkout or its index. +The conflict check creates a temporary repository and fetches the checked-out PR head locally. +The base and DB revisions come from `CBTS_COVERAGE_GIT_REPO`, bound to the same authenticated +`default-llm-repo` internal mirror used by normal CI checkouts. The public GitHub repository is +only a local-command fallback. The temporary repository never changes the CI checkout or its index. The compare API answers unless the revision has not reached the public mirror yet (404) or the token is missing (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is @@ -208,21 +211,26 @@ The PR base is `merge_base_commit.sha` from `main...`. The latest DB is freshness gate and telemetry. `ahead`, `behind`, and `identical` describe valid positions on main; diverged and unknown relations decline Tier 2. -The latest DB must accept the complete binary PR diff from base to head via a three-way `git apply` -with no conflicts. Conflict, fetch failure, or an unconstructable diff declines Tier 2 before the -large DB artifacts are downloaded. If it passes, `--coverage-max-drift` applies a second fail-closed +`commit-tree` represents the complete base-to-head PR change as one commit, and `cherry-pick` +tests that commit against the latest DB without serializing through patch format. If the DB predates +the PR base, a PR-only cherry-pick can conflict merely because it lacks intervening main changes. +That case retries a cumulative DB-to-head commit. The audit log calls out this compatibility diff +because it differs from the GitHub PR diff, and its DB-to-head changed files and per-file diffs +become the actual Tier 2 input. Other conflicts or unmeasurable checks decline before the large DB +artifacts are downloaded. If validation passes, `--coverage-max-drift` applies a second fail-closed bound: beyond 30 commits Tier 2 declines and the PR runs in full. ### 8.3 What happens with the result `--prepare DIR` does the whole fetch in one call: resolve the PR base, select the latest complete -pair, validate the PR diff, stream both tarballs down, unpack their identically named SQLite files -separately, and union them with `compact_db.merge_databases`. It writes the selection JSON beside -the merged SQLite as `cbts_coverage_db.json` and prints `{path, meta}`. Groovy is left with the two -things only it can do -— bind the credential and run `coverage_audit.py` over the result — and any failure anywhere is -caught and non-fatal: no prepared DB is returned, Tier 2 never runs, and the PR gets a full run. +pair, validate compatibility, stream both tarballs down, unpack their identically named SQLite +files separately, and union them with `compact_db.merge_databases`. It writes the selection JSON +beside the merged SQLite as `cbts_coverage_db.json` and prints the paths plus compatibility-diff +metadata. For a cumulative fallback, it also writes the corresponding Tier 2 changed-file payload. +Groovy audits the DB and logs which diff range will drive Tier 2 before running +`coverage_audit.py` over the result. Any failure anywhere is caught and non-fatal: no prepared DB is +returned, Tier 2 never runs, and the PR gets a full run. Those two paths reach `main.py` as `--coverage-db` and `--coverage-db-meta`, so a new selection field needs no Groovy change. `main.py` records all of it and **gates on the drift**: past diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index b9fa413458ac..cc1d2bd75b1e 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -14,19 +14,19 @@ """Resolve which post-merge CBTS touch DBs to use. Candidates are recent builds of `` with both x86 and SBSA -coverage tarballs. The newest complete pair wins. The PR diff must apply -cleanly to that DB's revision; otherwise Tier 2 declines rather than choosing -an older DB. Revision metadata comes from the forge compare API and needs -`GITHUB_API_TOKEN`, the anonymous quota being per-IP and exhausted by shared CI -egress. +coverage tarballs. The newest complete pair wins. A squashed PR commit must +cherry-pick cleanly to that DB's revision; when an older DB lacks intervening +main changes, a cumulative DB-to-head commit is checked instead. Revision +metadata comes from the forge compare API and needs `GITHUB_API_TOKEN`, the +anonymous quota being per-IP and exhausted by shared CI egress. The selected architecture DBs are merged locally through the compact coverage schema, producing the one DB consumed by `main.py`. -Two entry points. `--print-selection` prints `{urls, build, commit, lag, -base_commit, drift, drift_status, patch_apply_status}` and stops. `--prepare -DIR` goes on to download, unpack, and merge the winner, drop that JSON beside -it, and print `{path, meta}` — the two paths `main.py` needs. +Two entry points. `--print-selection` prints the selection metadata and stops. +`--prepare DIR` goes on to download, unpack, and merge the winner, drop that +JSON beside it, and print the paths and compatibility-diff metadata used by +the CI audit. """ from __future__ import annotations @@ -44,7 +44,7 @@ import urllib.request from functools import lru_cache from pathlib import Path -from typing import Literal, Optional +from typing import Literal, NamedTuple, Optional sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "coverage_utils")) @@ -59,6 +59,7 @@ # sqlite at the tar root, and the selection JSON `prepare` drops beside it. DB_NAME = "cbts_touchmap.sqlite" META_NAME = "cbts_coverage_db.json" +DIFF_INPUT_NAME = "cbts_coverage_diff.json" # Per-build metadata carrying `commit=`; absent on some builds. BUILD_INFO_NAME = "build_info.txt" @@ -66,6 +67,8 @@ COVERAGE_BRANCH = "main" # Read by `compare_distance`; the anonymous quota is unusable from shared CI egress IPs. GITHUB_TOKEN_ENV = "GITHUB_API_TOKEN" +# The normal CI checkout's authenticated repository URL, bound by Jenkins. +COVERAGE_GIT_REPO_ENV = "CBTS_COVERAGE_GIT_REPO" _URM = "https://urm.nvidia.com/artifactory" _GITHUB_COMPARE = "https://api.github.com/repos/NVIDIA/TensorRT-LLM/compare" @@ -85,6 +88,13 @@ _PatchApplyStatus = Literal["clean", "conflict", "unknown"] +class _PatchApplyResult(NamedTuple): + status: _PatchApplyStatus + diff_base_commit: str + changed_files: Optional[list[str]] = None + diffs: Optional[dict[str, str]] = None + + def _get(url: str, headers: Optional[dict] = None) -> tuple[Optional[int], Optional[bytes]]: req = urllib.request.Request(url, headers=headers or {}) try: @@ -235,15 +245,34 @@ def _run_git( return None +def _cumulative_diff(repo: Path, db_commit: str, pr_head: str) -> Optional[dict]: + """Build the DB-to-head changed-file payload consumed by Tier 2.""" + names = _run_git(["diff", "--name-only", "-z", "--find-renames", db_commit, pr_head], cwd=repo) + if names is None or names.returncode != 0: + return None + changed_files = [ + path for path in names.stdout.decode("utf-8", "surrogateescape").split("\0") if path + ] + diffs = {} + for path in changed_files: + diff = _run_git(["diff", "--find-renames", db_commit, pr_head, "--", path], cwd=repo) + if diff is None or diff.returncode != 0: + return None + diffs[path] = diff.stdout.decode("utf-8", "replace") + return {"changed_files": changed_files, "diffs": diffs} + + def _patch_apply_status( pr_base_commit: str, pr_head: str, db_commit: str, repo_root: Path = Path("."), - upstream_url: str = _GITHUB_GIT_REPO, -) -> _PatchApplyStatus: - """Return clean/conflict/unknown for applying the PR diff to the DB revision.""" + upstream_url: Optional[str] = None, + db_drift_status: str = "", +) -> _PatchApplyResult: + """Check a squashed PR commit against the DB revision.""" repo_source = repo_root.resolve() + main_repo = upstream_url or os.environ.get(COVERAGE_GIT_REPO_ENV) or _GITHUB_GIT_REPO checked_out_head = _run_git(["rev-parse", "HEAD"], cwd=repo_source) if ( checked_out_head is None @@ -251,111 +280,136 @@ def _patch_apply_status( or checked_out_head.stdout.decode("ascii", "replace").strip() != pr_head ): print(f"[artifact] checked-out revision is not PR head {pr_head[:10]}", file=sys.stderr) - return "unknown" + return _PatchApplyResult("unknown", pr_base_commit) with tempfile.TemporaryDirectory(prefix="cbts_patch_check_") as temp_dir: temp = Path(temp_dir) - git_dir = temp / "repo.git" - initialized = _run_git(["init", "--bare", str(git_dir)], cwd=temp) + repo = temp / "repo" + initialized = _run_git(["init", str(repo)], cwd=temp) if initialized is None or initialized.returncode != 0: print("[artifact] could not initialize the patch-check repository", file=sys.stderr) - return "unknown" + return _PatchApplyResult("unknown", pr_base_commit) - # Fetching into a temporary bare repository leaves the shallow CI checkout and its - # index untouched. The checked-out PR head is available locally; both main revisions - # are fetched from the public upstream repository. + # The PR head is guaranteed to be the checked-out revision. The base and coverage + # commits come from the same authenticated internal mirror as normal CI checkouts. fetched_head = _run_git( - [ - "--git-dir", - str(git_dir), - "fetch", - "--no-tags", - "--depth=1", - str(repo_source), - "HEAD", - ], - cwd=temp, + ["fetch", "--no-tags", "--depth=1", str(repo_source), "HEAD"], cwd=repo ) if fetched_head is None or fetched_head.returncode != 0: print( f"[artifact] could not load PR head {pr_head[:10]} for patch check", file=sys.stderr, ) - return "unknown" + return _PatchApplyResult("unknown", pr_base_commit) revisions = list(dict.fromkeys((pr_base_commit, db_commit))) fetched_revisions = _run_git( - [ - "--git-dir", - str(git_dir), - "fetch", - "--no-tags", - "--depth=1", - upstream_url, - *revisions, - ], - cwd=temp, + ["fetch", "--no-tags", "--depth=1", main_repo, *revisions], cwd=repo ) if fetched_revisions is None or fetched_revisions.returncode != 0: print( "[artifact] could not load the base/coverage revisions for patch check", file=sys.stderr, ) - return "unknown" + return _PatchApplyResult("unknown", pr_base_commit) + + checked_out_db = _run_git(["checkout", "--detach", db_commit], cwd=repo) + if checked_out_db is None or checked_out_db.returncode != 0: + print("[artifact] could not check out the coverage revision", file=sys.stderr) + return _PatchApplyResult("unknown", pr_base_commit) - diff = _run_git( + squashed_pr = _run_git( [ - "--git-dir", - str(git_dir), - "diff", - "--binary", - "--full-index", + "-c", + "user.name=CBTS", + "-c", + "user.email=cbts@nvidia.com", + "commit-tree", + f"{pr_head}^{{tree}}", + "-p", pr_base_commit, - pr_head, + "-m", + "squashed PR for CBTS compatibility check", ], - cwd=temp, - ) - if diff is None or diff.returncode != 0: - print("[artifact] could not generate the PR diff for patch check", file=sys.stderr) - return "unknown" - - index = temp / "index" - git_env = os.environ.copy() - git_env["GIT_INDEX_FILE"] = str(index) - read_tree = _run_git( - ["--git-dir", str(git_dir), "read-tree", db_commit], cwd=temp, env=git_env + cwd=repo, ) - if read_tree is None or read_tree.returncode != 0: - print( - "[artifact] could not read the coverage revision for patch check", - file=sys.stderr, - ) - return "unknown" + if squashed_pr is None or squashed_pr.returncode != 0: + print("[artifact] could not create the squashed PR commit", file=sys.stderr) + return _PatchApplyResult("unknown", pr_base_commit) - applied = _run_git( + pr_commit = squashed_pr.stdout.decode("ascii", "replace").strip() + applied = _run_git(["cherry-pick", "--no-commit", pr_commit], cwd=repo) + if applied is None: + return _PatchApplyResult("unknown", pr_base_commit) + if applied.returncode == 0: + return _PatchApplyResult("clean", pr_base_commit) + if db_drift_status != "ahead": + return _PatchApplyResult("conflict", pr_base_commit) + + # An older DB can lack main changes that the PR patch assumes. Retry the full + # DB-to-head delta so those prerequisites are included instead of misclassifying + # ordinary staleness as a PR conflict. + print( + f"[artifact] PR diff {pr_base_commit[:10]}..{pr_head[:10]} conflicts with older " + f"DB {db_commit[:10]}; retrying cumulative diff {db_commit[:10]}..{pr_head[:10]}", + file=sys.stderr, + ) + reset = _run_git(["reset", "--hard", db_commit], cwd=repo) + if reset is None or reset.returncode != 0: + return _PatchApplyResult("unknown", db_commit) + cumulative = _run_git( [ - "--git-dir", - str(git_dir), - "apply", - "--cached", - "--3way", - "--whitespace=nowarn", - "-", + "-c", + "user.name=CBTS", + "-c", + "user.email=cbts@nvidia.com", + "commit-tree", + f"{pr_head}^{{tree}}", + "-p", + db_commit, + "-m", + "cumulative diff for CBTS compatibility check", ], - cwd=temp, - input_data=diff.stdout, - env=git_env, + cwd=repo, ) + if cumulative is None or cumulative.returncode != 0: + return _PatchApplyResult("unknown", db_commit) + cumulative_commit = cumulative.stdout.decode("ascii", "replace").strip() + applied = _run_git(["cherry-pick", "--no-commit", cumulative_commit], cwd=repo) if applied is None: - return "unknown" - return "clean" if applied.returncode == 0 else "conflict" + return _PatchApplyResult("unknown", db_commit) + status: _PatchApplyStatus = "clean" if applied.returncode == 0 else "conflict" + if status != "clean": + return _PatchApplyResult(status, db_commit) + cumulative_diff = _cumulative_diff(repo, db_commit, pr_head) + if cumulative_diff is None: + return _PatchApplyResult("unknown", db_commit) + return _PatchApplyResult( + "clean", + db_commit, + cumulative_diff["changed_files"], + cumulative_diff["diffs"], + ) def _selection_accepts_pr_diff(sel: dict, pr_base_commit: str, pr_head: str) -> bool: - sel["patch_apply_status"] = _patch_apply_status(pr_base_commit, pr_head, sel["commit"]) + patch = _patch_apply_status( + pr_base_commit, + pr_head, + sel["commit"], + db_drift_status=sel.get("drift_status") or "", + ) + sel["patch_apply_status"] = patch.status + sel["patch_diff_base_commit"] = patch.diff_base_commit + sel["patch_diff_head_commit"] = pr_head + sel["patch_diff_kind"] = ( + "cumulative" if patch.diff_base_commit == sel["commit"] != pr_base_commit else "pr" + ) + if patch.changed_files is not None and patch.diffs is not None: + sel["_patch_input"] = {"changed_files": patch.changed_files, "diffs": patch.diffs} if sel["patch_apply_status"] == "clean": return True print( - f"[artifact] coverage tier declined: PR diff does not apply cleanly to " + f"[artifact] coverage tier declined: compatibility diff does not apply cleanly to " f"latest DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})", file=sys.stderr, ) @@ -438,11 +492,20 @@ def describe(sel: dict) -> str: else: base_distance = f"+{sel['drift']} commit(s)" base = (sel.get("base_commit") or "unknown")[:10] - return ( + selection = ( f"[artifact] selected build {sel.get('build')}, " f"DB commit {(sel.get('commit') or 'unknown')[:10]}; topology from DB: " f"PR base {base} ({base_distance}), current main tip ({lag})" ) + diff_base = (sel.get("patch_diff_base_commit") or "unknown")[:10] + diff_head = (sel.get("patch_diff_head_commit") or "unknown")[:10] + if sel.get("patch_diff_kind") == "cumulative": + diff_description = ( + f"cumulative {diff_base}..{diff_head}; differs from GitHub PR diff {base}..{diff_head}" + ) + else: + diff_description = f"PR {diff_base}..{diff_head}" + return f"{selection}; compatibility diff: {diff_description}" def download(url: str, dest: Path, attempts: int = _RETRIES) -> Optional[Path]: @@ -524,8 +587,21 @@ def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: connection.close() meta = dest / META_NAME + patch_input = sel.pop("_patch_input", None) meta.write_text(json.dumps(sel)) - return {"path": str(db), "meta": str(meta)} + ready = { + "path": str(db), + "meta": str(meta), + "patch_diff_kind": sel["patch_diff_kind"], + "patch_diff_base_commit": sel["patch_diff_base_commit"], + "patch_diff_head_commit": sel["patch_diff_head_commit"], + "pr_base_commit": pr_base_commit, + } + if patch_input is not None: + diff_input = dest / DIFF_INPUT_NAME + diff_input.write_text(json.dumps(patch_input)) + ready["patch_input"] = str(diff_input) + return ready def main(argv: Optional[list[str]] = None) -> int: @@ -545,8 +621,8 @@ def main(argv: Optional[list[str]] = None) -> int: "--prepare", metavar="DIR", default=None, - help="resolve, download and merge the architecture DBs into DIR, then print " - "{path, meta} as JSON", + help="resolve, validate, download and merge the architecture DBs into DIR, then " + "print paths and compatibility-diff metadata as JSON", ) ap.add_argument( "--pr-head", @@ -594,6 +670,7 @@ def main(argv: Optional[list[str]] = None) -> int: return 1 if not _selection_accepts_pr_diff(best, pr_base_commit, args.pr_head): return 1 + best.pop("_patch_input", None) print(json.dumps(best)) return 0 diff --git a/tests/unittest/scripts/test_cbts_coverage_artifact.py b/tests/unittest/scripts/test_cbts_coverage_artifact.py index 0f977150d285..f34a8016569b 100644 --- a/tests/unittest/scripts/test_cbts_coverage_artifact.py +++ b/tests/unittest/scripts/test_cbts_coverage_artifact.py @@ -67,9 +67,13 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: _git(repo, "config", "user.email", "cbts@example.com") _git(repo, "config", "user.name", "CBTS Test") source = repo / "source.py" - source.write_text("first\nbase\nlast\n") + source.write_text("first\nold\nlast\n") _git(repo, "add", "source.py") - _git(repo, "commit", "-m", "base") + _git(repo, "commit", "-m", "older coverage revision") + older_db = _git(repo, "rev-parse", "HEAD") + + source.write_text("first\nbase\nlast\n") + _git(repo, "commit", "-am", "base") base = _git(repo, "rev-parse", "HEAD") _git(repo, "checkout", "-b", "pr") @@ -90,12 +94,22 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: _git(repo, "checkout", "pr") self.assertEqual( - artifact._patch_apply_status(base, head, clean_db, repo, str(repo)), "clean" + artifact._patch_apply_status(base, head, clean_db, repo, str(repo)).status, + "clean", ) self.assertEqual( - artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)), + artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)).status, "conflict", ) + cumulative = artifact._patch_apply_status( + base, head, older_db, repo, str(repo), db_drift_status="ahead" + ) + self.assertEqual(cumulative.status, "clean") + self.assertEqual(cumulative.diff_base_commit, older_db) + self.assertEqual(cumulative.changed_files, ["source.py"]) + assert cumulative.diffs is not None + self.assertIn("-old", cumulative.diffs["source.py"]) + self.assertIn("+pr", cumulative.diffs["source.py"]) def test_accepts_artifact_collected_at_pr_base(self) -> None: with ( @@ -165,7 +179,11 @@ def extract(tarball: Path, destination: Path) -> bool: with ( mock.patch.object(artifact, "merge_base", return_value="pr-base"), mock.patch.object(artifact, "select_tarball", return_value=selection) as select, - mock.patch.object(artifact, "_patch_apply_status", return_value="clean") as apply, + mock.patch.object( + artifact, + "_patch_apply_status", + return_value=artifact._PatchApplyResult("clean", "pr-base"), + ) as apply, mock.patch.object(artifact, "download", side_effect=download), mock.patch.object(artifact, "extract", side_effect=extract), ): @@ -174,7 +192,9 @@ def extract(tarball: Path, destination: Path) -> bool: self.assertIsNotNone(ready) assert ready is not None select.assert_called_once_with("pr-base") - apply.assert_called_once_with("pr-base", "pr-head", "coverage-commit") + apply.assert_called_once_with( + "pr-base", "pr-head", "coverage-commit", db_drift_status="ahead" + ) connection = sqlite3.connect(ready["path"]) try: tests = { From d01eb6ea55fe4cb9931da43182ebc7fb1bce01a1 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:23:41 +0800 Subject: [PATCH 05/10] [TRTLLM-15885][infra] isolate coverage tier diff input Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 28 +++++++++++++------ jenkins/scripts/cbts/README.md | 4 +-- .../cbts/coverage_selection/SELECTION.md | 12 ++++---- jenkins/scripts/cbts/main.py | 16 ++++++++++- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index b5d593f269b4..96021c1e30ed 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -967,7 +967,6 @@ def getCbtsResult(pipeline, testFilter, globalVars) if (coverageDb?.patch_input) { coveragePatchInput = new groovy.json.JsonSlurper().parseText( readFile("${LLM_ROOT}/${coverageDb.patch_input}")) - changedFiles = (coveragePatchInput.changed_files ?: []).unique() } // Ask Python which file patterns need diffs, fetch them. @@ -981,13 +980,10 @@ def getCbtsResult(pipeline, testFilter, globalVars) } def diffs = [:] if (filesNeedingDiff) { - def fileChanges = coveragePatchInput?.diffs - if (fileChanges == null) { - def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] - fileChanges = githubPrApiUrl != null - ? getGithubMRChangedFile(pipeline, githubPrApiUrl, "getFileChanges") - : getGitlabMRChangedFile(pipeline, "getFileChanges") - } + def githubPrApiUrl = globalVars[GITHUB_PR_API_URL] + def fileChanges = githubPrApiUrl != null + ? getGithubMRChangedFile(pipeline, githubPrApiUrl, "getFileChanges") + : getGitlabMRChangedFile(pipeline, "getFileChanges") diffs = filesNeedingDiff.collectEntries { filePath -> // Null (patch omitted for binary / rename / too-large diffs) coerces to empty. [(filePath): fileChanges[filePath] ?: ""] @@ -1007,6 +1003,19 @@ def getCbtsResult(pipeline, testFilter, globalVars) if (coverageDb) { mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-meta ${coverageDb.meta}" } + if (coveragePatchInput) { + def coverageDiffs = filesNeedingDiff.collectEntries { filePath -> + [(filePath): coveragePatchInput.diffs[filePath] ?: ""] + } + def coverageInputJson = groovy.json.JsonOutput.toJson([ + changed_files: changedFiles, + diffs: coverageDiffs, + post_merge: testFilter[(IS_POST_MERGE)] ?: false, + ]) + def coverageInputPath = "${LLM_ROOT}/cbts_coverage_input.json" + writeFile file: coverageInputPath, text: coverageInputJson + mainCmd += " --coverage-input cbts_coverage_input.json" + } def output = sh(script: mainCmd, returnStdout: true) def result = _cbtsParseSelectionResult(output) @@ -1113,7 +1122,8 @@ def _cbtsCoverageAudit(pipeline) } def ready = new groovy.json.JsonSlurper().parseText(readyJson) if (ready.patch_diff_kind == "cumulative") { - pipeline.echo("CBTS audit: Tier 2 will use cumulative diff " + + pipeline.echo("CBTS audit: Tier 2 will evaluate the current PR files using " + + "cumulative per-file diffs from " + "${ready.patch_diff_base_commit.take(10)}.." + "${ready.patch_diff_head_commit.take(10)}, not the GitHub PR diff " + "${ready.pr_base_commit.take(10)}.." + diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 4c0948716801..e90b85a200af 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -327,8 +327,8 @@ CBTS defers to the existing filter chain when: - No touch DB artifact could be resolved — Tier 2 never runs - Neither the current PR diff nor its permitted older-DB cumulative fallback can be applied to the latest complete coverage DB revision (or the check cannot be completed) — Tier 2 declines before - download. When that fallback is used successfully, Tier 2 uses its DB-to-PR-head changed files - and diffs, and the audit log warns that this input can differ from the diff shown on the PR + download. When that fallback is used successfully, Tier 1 keeps the PR diff while Tier 2 uses + DB-to-PR-head diffs for the PR's files; the audit log warns that these can differ from the PR - The resolved DB sits more than `--coverage-max-drift` commits from the PR's base commit, on either side, or an unmeasurable distance from it — Tier 2 declines (`coverage_freshness` = `stale` / `unknown`) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index c1853903fc9b..8e5c8a59e6ca 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -215,10 +215,11 @@ diverged and unknown relations decline Tier 2. tests that commit against the latest DB without serializing through patch format. If the DB predates the PR base, a PR-only cherry-pick can conflict merely because it lacks intervening main changes. That case retries a cumulative DB-to-head commit. The audit log calls out this compatibility diff -because it differs from the GitHub PR diff, and its DB-to-head changed files and per-file diffs -become the actual Tier 2 input. Other conflicts or unmeasurable checks decline before the large DB -artifacts are downloaded. If validation passes, `--coverage-max-drift` applies a second fail-closed -bound: beyond 30 commits Tier 2 declines and the PR runs in full. +because it differs from the GitHub PR diff. Tier 1 keeps the forge PR payload. Tier 2 keeps the PR's +changed-file set but uses the corresponding DB-to-head per-file diffs; unrelated files changed only +between the DB and PR base do not enter either tier. Other conflicts or unmeasurable checks decline +before the large DB artifacts are downloaded. If validation passes, `--coverage-max-drift` applies +a second fail-closed bound: beyond 30 commits Tier 2 declines and the PR runs in full. ### 8.3 What happens with the result @@ -228,7 +229,8 @@ pair, validate compatibility, stream both tarballs down, unpack their identicall files separately, and union them with `compact_db.merge_databases`. It writes the selection JSON beside the merged SQLite as `cbts_coverage_db.json` and prints the paths plus compatibility-diff metadata. For a cumulative fallback, it also writes the corresponding Tier 2 changed-file payload. -Groovy audits the DB and logs which diff range will drive Tier 2 before running +Groovy intersects its per-file diffs with the PR changed-file set, passes that as a separate +`--coverage-input`, and logs which diff range will drive Tier 2 before running `coverage_audit.py` over the result. Any failure anywhere is caught and non-fatal: no prepared DB is returned, Tier 2 never runs, and the PR gets a full run. diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index c925359bdb11..22a0f93511f5 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -395,6 +395,12 @@ def main(argv: Optional[list[str]] = None) -> int: "--coverage-db is. Its `drift` is what the freshness gate decides on; the " "rest is recorded. Absent or unreadable declines the tier.", ) + parser.add_argument( + "--coverage-input", + default=None, + help="Optional INPUT_JSON used only by the coverage tier. Tier 1 always uses the " + "positional PR input.", + ) parser.add_argument( "--coverage-max-drift", type=int, @@ -472,9 +478,17 @@ def main(argv: Optional[list[str]] = None) -> int: ) if not note: # the gate passed; a note here means it did not try: + coverage_pr = pr + if args.coverage_input: + coverage_diff_input = _load_pr_inputs(Path(args.coverage_input)) + coverage_pr = PRInputs( + changed_files=pr.changed_files, + diffs=coverage_diff_input.diffs, + post_merge=pr.post_merge, + ) db = open_db(args.coverage_db) tier, note = apply_coverage_tier( - pr, + coverage_pr, selector.pairs, selector.handled, stages, From a8f21e864a547537ddfd096ac3852b5c7e5e05b6 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:50:51 +0800 Subject: [PATCH 06/10] [TRTLLM-15885][infra] simplify coverage compatibility gate Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 32 +--- jenkins/scripts/cbts/README.md | 6 +- .../cbts/coverage_selection/SELECTION.md | 26 ++- .../cbts/coverage_selection/artifact.py | 149 +++--------------- jenkins/scripts/cbts/main.py | 16 +- .../scripts/test_cbts_coverage_artifact.py | 32 +--- 6 files changed, 40 insertions(+), 221 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 96021c1e30ed..cfc7ea28c871 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -963,11 +963,6 @@ def getCbtsResult(pipeline, testFilter, globalVars) // Download the touch DB only for PRs in the coverage-tier pilot. def coverageDb = _cbtsCoverageAudit(pipeline) - def coveragePatchInput = null - if (coverageDb?.patch_input) { - coveragePatchInput = new groovy.json.JsonSlurper().parseText( - readFile("${LLM_ROOT}/${coverageDb.patch_input}")) - } // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( @@ -1003,19 +998,6 @@ def getCbtsResult(pipeline, testFilter, globalVars) if (coverageDb) { mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-meta ${coverageDb.meta}" } - if (coveragePatchInput) { - def coverageDiffs = filesNeedingDiff.collectEntries { filePath -> - [(filePath): coveragePatchInput.diffs[filePath] ?: ""] - } - def coverageInputJson = groovy.json.JsonOutput.toJson([ - changed_files: changedFiles, - diffs: coverageDiffs, - post_merge: testFilter[(IS_POST_MERGE)] ?: false, - ]) - def coverageInputPath = "${LLM_ROOT}/cbts_coverage_input.json" - writeFile file: coverageInputPath, text: coverageInputJson - mainCmd += " --coverage-input cbts_coverage_input.json" - } def output = sh(script: mainCmd, returnStdout: true) def result = _cbtsParseSelectionResult(output) @@ -1080,7 +1062,7 @@ def _cbtsMultiGpuLabelGateOpen(pipeline, globalVars) } // Check pilot eligibility, then fetch and audit the touch DB; artifact.py's -// prepared paths and compatibility-diff metadata verbatim, or null on failure. +// {path, meta} verbatim, or null on failure. def _cbtsCoverageAudit(pipeline) { try { @@ -1121,17 +1103,7 @@ def _cbtsCoverageAudit(pipeline) return null } def ready = new groovy.json.JsonSlurper().parseText(readyJson) - if (ready.patch_diff_kind == "cumulative") { - pipeline.echo("CBTS audit: Tier 2 will evaluate the current PR files using " + - "cumulative per-file diffs from " + - "${ready.patch_diff_base_commit.take(10)}.." + - "${ready.patch_diff_head_commit.take(10)}, not the GitHub PR diff " + - "${ready.pr_base_commit.take(10)}.." + - "${ready.patch_diff_head_commit.take(10)}") - } else { - pipeline.echo("CBTS audit: coverage compatibility used GitHub PR diff " + - "${ready.patch_diff_base_commit.take(10)}..${ready.patch_diff_head_commit.take(10)}") - } + pipeline.echo("CBTS audit: PR diff applies cleanly to the latest coverage DB") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}" return ready } catch (InterruptedException e) { diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index e90b85a200af..eb39f3440b65 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -325,10 +325,8 @@ CBTS defers to the existing filter chain when: decorator line), has no usable patch, has unparsable source, or has a closure change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs -- Neither the current PR diff nor its permitted older-DB cumulative fallback can be applied to the - latest complete coverage DB revision (or the check cannot be completed) — Tier 2 declines before - download. When that fallback is used successfully, Tier 1 keeps the PR diff while Tier 2 uses - DB-to-PR-head diffs for the PR's files; the audit log warns that these can differ from the PR +- The current PR diff cannot be applied without conflicts to the latest complete coverage DB + revision (or the check cannot be completed) — Tier 2 declines before download - The resolved DB sits more than `--coverage-max-drift` commits from the PR's base commit, on either side, or an unmeasurable distance from it — Tier 2 declines (`coverage_freshness` = `stale` / `unknown`) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 8e5c8a59e6ca..18a093d0abcb 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -167,8 +167,7 @@ fetch PR head locally; fetch base and latest DB from the normal CI Git mirror create a squashed PR commit with `commit-tree`, parented at the PR base cherry-pick it onto the DB revision clean → continue - conflict and DB is older than PR base → retry cumulative DB-to-head commit - other conflict / unavailable revision → decline Tier 2 + conflict / unavailable revision → decline Tier 2 ``` Requiring the pair prevents the selector from narrowing only one CPU architecture. Builds are @@ -212,14 +211,11 @@ freshness gate and telemetry. `ahead`, `behind`, and `identical` describe valid diverged and unknown relations decline Tier 2. `commit-tree` represents the complete base-to-head PR change as one commit, and `cherry-pick` -tests that commit against the latest DB without serializing through patch format. If the DB predates -the PR base, a PR-only cherry-pick can conflict merely because it lacks intervening main changes. -That case retries a cumulative DB-to-head commit. The audit log calls out this compatibility diff -because it differs from the GitHub PR diff. Tier 1 keeps the forge PR payload. Tier 2 keeps the PR's -changed-file set but uses the corresponding DB-to-head per-file diffs; unrelated files changed only -between the DB and PR base do not enter either tier. Other conflicts or unmeasurable checks decline -before the large DB artifacts are downloaded. If validation passes, `--coverage-max-drift` applies -a second fail-closed bound: beyond 30 commits Tier 2 declines and the PR runs in full. +tests that commit against the latest DB without serializing through patch format. This check is +independent of whether the DB is older than, equal to, or newer than the PR base. A conflict or an +unmeasurable check declines before the large DB artifacts are downloaded. If it passes, Tier 2 uses +the original forge PR payload. `--coverage-max-drift` applies a second fail-closed bound: beyond 30 +commits Tier 2 declines and the PR runs in full. ### 8.3 What happens with the result @@ -227,12 +223,10 @@ a second fail-closed bound: beyond 30 commits Tier 2 declines and the PR runs in `--prepare DIR` does the whole fetch in one call: resolve the PR base, select the latest complete pair, validate compatibility, stream both tarballs down, unpack their identically named SQLite files separately, and union them with `compact_db.merge_databases`. It writes the selection JSON -beside the merged SQLite as `cbts_coverage_db.json` and prints the paths plus compatibility-diff -metadata. For a cumulative fallback, it also writes the corresponding Tier 2 changed-file payload. -Groovy intersects its per-file diffs with the PR changed-file set, passes that as a separate -`--coverage-input`, and logs which diff range will drive Tier 2 before running -`coverage_audit.py` over the result. Any failure anywhere is caught and non-fatal: no prepared DB is -returned, Tier 2 never runs, and the PR gets a full run. +beside the merged SQLite as `cbts_coverage_db.json` and prints `{path, meta}`. Groovy binds the +credentials, logs the successful compatibility check, and runs `coverage_audit.py` over the result. +Any failure anywhere is caught and non-fatal: no prepared DB is returned, Tier 2 never runs, and the +PR gets a full run. Those two paths reach `main.py` as `--coverage-db` and `--coverage-db-meta`, so a new selection field needs no Groovy change. `main.py` records all of it and **gates on the drift**: past diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index cc1d2bd75b1e..a2d57c38c901 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -15,8 +15,7 @@ Candidates are recent builds of `` with both x86 and SBSA coverage tarballs. The newest complete pair wins. A squashed PR commit must -cherry-pick cleanly to that DB's revision; when an older DB lacks intervening -main changes, a cumulative DB-to-head commit is checked instead. Revision +cherry-pick cleanly to that DB's revision; otherwise Tier 2 declines. Revision metadata comes from the forge compare API and needs `GITHUB_API_TOKEN`, the anonymous quota being per-IP and exhausted by shared CI egress. @@ -25,8 +24,7 @@ Two entry points. `--print-selection` prints the selection metadata and stops. `--prepare DIR` goes on to download, unpack, and merge the winner, drop that -JSON beside it, and print the paths and compatibility-diff metadata used by -the CI audit. +JSON beside it, and print the two paths consumed by CBTS. """ from __future__ import annotations @@ -44,7 +42,7 @@ import urllib.request from functools import lru_cache from pathlib import Path -from typing import Literal, NamedTuple, Optional +from typing import Literal, Optional sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "coverage_utils")) @@ -59,7 +57,6 @@ # sqlite at the tar root, and the selection JSON `prepare` drops beside it. DB_NAME = "cbts_touchmap.sqlite" META_NAME = "cbts_coverage_db.json" -DIFF_INPUT_NAME = "cbts_coverage_diff.json" # Per-build metadata carrying `commit=`; absent on some builds. BUILD_INFO_NAME = "build_info.txt" @@ -88,13 +85,6 @@ _PatchApplyStatus = Literal["clean", "conflict", "unknown"] -class _PatchApplyResult(NamedTuple): - status: _PatchApplyStatus - diff_base_commit: str - changed_files: Optional[list[str]] = None - diffs: Optional[dict[str, str]] = None - - def _get(url: str, headers: Optional[dict] = None) -> tuple[Optional[int], Optional[bytes]]: req = urllib.request.Request(url, headers=headers or {}) try: @@ -245,31 +235,13 @@ def _run_git( return None -def _cumulative_diff(repo: Path, db_commit: str, pr_head: str) -> Optional[dict]: - """Build the DB-to-head changed-file payload consumed by Tier 2.""" - names = _run_git(["diff", "--name-only", "-z", "--find-renames", db_commit, pr_head], cwd=repo) - if names is None or names.returncode != 0: - return None - changed_files = [ - path for path in names.stdout.decode("utf-8", "surrogateescape").split("\0") if path - ] - diffs = {} - for path in changed_files: - diff = _run_git(["diff", "--find-renames", db_commit, pr_head, "--", path], cwd=repo) - if diff is None or diff.returncode != 0: - return None - diffs[path] = diff.stdout.decode("utf-8", "replace") - return {"changed_files": changed_files, "diffs": diffs} - - def _patch_apply_status( pr_base_commit: str, pr_head: str, db_commit: str, repo_root: Path = Path("."), upstream_url: Optional[str] = None, - db_drift_status: str = "", -) -> _PatchApplyResult: +) -> _PatchApplyStatus: """Check a squashed PR commit against the DB revision.""" repo_source = repo_root.resolve() main_repo = upstream_url or os.environ.get(COVERAGE_GIT_REPO_ENV) or _GITHUB_GIT_REPO @@ -280,14 +252,14 @@ def _patch_apply_status( or checked_out_head.stdout.decode("ascii", "replace").strip() != pr_head ): print(f"[artifact] checked-out revision is not PR head {pr_head[:10]}", file=sys.stderr) - return _PatchApplyResult("unknown", pr_base_commit) + return "unknown" with tempfile.TemporaryDirectory(prefix="cbts_patch_check_") as temp_dir: temp = Path(temp_dir) repo = temp / "repo" initialized = _run_git(["init", str(repo)], cwd=temp) if initialized is None or initialized.returncode != 0: print("[artifact] could not initialize the patch-check repository", file=sys.stderr) - return _PatchApplyResult("unknown", pr_base_commit) + return "unknown" # The PR head is guaranteed to be the checked-out revision. The base and coverage # commits come from the same authenticated internal mirror as normal CI checkouts. @@ -299,7 +271,7 @@ def _patch_apply_status( f"[artifact] could not load PR head {pr_head[:10]} for patch check", file=sys.stderr, ) - return _PatchApplyResult("unknown", pr_base_commit) + return "unknown" revisions = list(dict.fromkeys((pr_base_commit, db_commit))) fetched_revisions = _run_git( @@ -310,12 +282,12 @@ def _patch_apply_status( "[artifact] could not load the base/coverage revisions for patch check", file=sys.stderr, ) - return _PatchApplyResult("unknown", pr_base_commit) + return "unknown" checked_out_db = _run_git(["checkout", "--detach", db_commit], cwd=repo) if checked_out_db is None or checked_out_db.returncode != 0: print("[artifact] could not check out the coverage revision", file=sys.stderr) - return _PatchApplyResult("unknown", pr_base_commit) + return "unknown" squashed_pr = _run_git( [ @@ -334,82 +306,21 @@ def _patch_apply_status( ) if squashed_pr is None or squashed_pr.returncode != 0: print("[artifact] could not create the squashed PR commit", file=sys.stderr) - return _PatchApplyResult("unknown", pr_base_commit) + return "unknown" pr_commit = squashed_pr.stdout.decode("ascii", "replace").strip() applied = _run_git(["cherry-pick", "--no-commit", pr_commit], cwd=repo) if applied is None: - return _PatchApplyResult("unknown", pr_base_commit) - if applied.returncode == 0: - return _PatchApplyResult("clean", pr_base_commit) - if db_drift_status != "ahead": - return _PatchApplyResult("conflict", pr_base_commit) - - # An older DB can lack main changes that the PR patch assumes. Retry the full - # DB-to-head delta so those prerequisites are included instead of misclassifying - # ordinary staleness as a PR conflict. - print( - f"[artifact] PR diff {pr_base_commit[:10]}..{pr_head[:10]} conflicts with older " - f"DB {db_commit[:10]}; retrying cumulative diff {db_commit[:10]}..{pr_head[:10]}", - file=sys.stderr, - ) - reset = _run_git(["reset", "--hard", db_commit], cwd=repo) - if reset is None or reset.returncode != 0: - return _PatchApplyResult("unknown", db_commit) - cumulative = _run_git( - [ - "-c", - "user.name=CBTS", - "-c", - "user.email=cbts@nvidia.com", - "commit-tree", - f"{pr_head}^{{tree}}", - "-p", - db_commit, - "-m", - "cumulative diff for CBTS compatibility check", - ], - cwd=repo, - ) - if cumulative is None or cumulative.returncode != 0: - return _PatchApplyResult("unknown", db_commit) - cumulative_commit = cumulative.stdout.decode("ascii", "replace").strip() - applied = _run_git(["cherry-pick", "--no-commit", cumulative_commit], cwd=repo) - if applied is None: - return _PatchApplyResult("unknown", db_commit) - status: _PatchApplyStatus = "clean" if applied.returncode == 0 else "conflict" - if status != "clean": - return _PatchApplyResult(status, db_commit) - cumulative_diff = _cumulative_diff(repo, db_commit, pr_head) - if cumulative_diff is None: - return _PatchApplyResult("unknown", db_commit) - return _PatchApplyResult( - "clean", - db_commit, - cumulative_diff["changed_files"], - cumulative_diff["diffs"], - ) + return "unknown" + return "clean" if applied.returncode == 0 else "conflict" def _selection_accepts_pr_diff(sel: dict, pr_base_commit: str, pr_head: str) -> bool: - patch = _patch_apply_status( - pr_base_commit, - pr_head, - sel["commit"], - db_drift_status=sel.get("drift_status") or "", - ) - sel["patch_apply_status"] = patch.status - sel["patch_diff_base_commit"] = patch.diff_base_commit - sel["patch_diff_head_commit"] = pr_head - sel["patch_diff_kind"] = ( - "cumulative" if patch.diff_base_commit == sel["commit"] != pr_base_commit else "pr" - ) - if patch.changed_files is not None and patch.diffs is not None: - sel["_patch_input"] = {"changed_files": patch.changed_files, "diffs": patch.diffs} + sel["patch_apply_status"] = _patch_apply_status(pr_base_commit, pr_head, sel["commit"]) if sel["patch_apply_status"] == "clean": return True print( - f"[artifact] coverage tier declined: compatibility diff does not apply cleanly to " + f"[artifact] coverage tier declined: PR diff does not apply cleanly to " f"latest DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})", file=sys.stderr, ) @@ -492,20 +403,12 @@ def describe(sel: dict) -> str: else: base_distance = f"+{sel['drift']} commit(s)" base = (sel.get("base_commit") or "unknown")[:10] - selection = ( + return ( f"[artifact] selected build {sel.get('build')}, " f"DB commit {(sel.get('commit') or 'unknown')[:10]}; topology from DB: " - f"PR base {base} ({base_distance}), current main tip ({lag})" + f"PR base {base} ({base_distance}), current main tip ({lag}); " + f"PR diff compatibility: {sel.get('patch_apply_status', 'unknown')}" ) - diff_base = (sel.get("patch_diff_base_commit") or "unknown")[:10] - diff_head = (sel.get("patch_diff_head_commit") or "unknown")[:10] - if sel.get("patch_diff_kind") == "cumulative": - diff_description = ( - f"cumulative {diff_base}..{diff_head}; differs from GitHub PR diff {base}..{diff_head}" - ) - else: - diff_description = f"PR {diff_base}..{diff_head}" - return f"{selection}; compatibility diff: {diff_description}" def download(url: str, dest: Path, attempts: int = _RETRIES) -> Optional[Path]: @@ -587,21 +490,8 @@ def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: connection.close() meta = dest / META_NAME - patch_input = sel.pop("_patch_input", None) meta.write_text(json.dumps(sel)) - ready = { - "path": str(db), - "meta": str(meta), - "patch_diff_kind": sel["patch_diff_kind"], - "patch_diff_base_commit": sel["patch_diff_base_commit"], - "patch_diff_head_commit": sel["patch_diff_head_commit"], - "pr_base_commit": pr_base_commit, - } - if patch_input is not None: - diff_input = dest / DIFF_INPUT_NAME - diff_input.write_text(json.dumps(patch_input)) - ready["patch_input"] = str(diff_input) - return ready + return {"path": str(db), "meta": str(meta)} def main(argv: Optional[list[str]] = None) -> int: @@ -622,7 +512,7 @@ def main(argv: Optional[list[str]] = None) -> int: metavar="DIR", default=None, help="resolve, validate, download and merge the architecture DBs into DIR, then " - "print paths and compatibility-diff metadata as JSON", + "print {path, meta} as JSON", ) ap.add_argument( "--pr-head", @@ -670,7 +560,6 @@ def main(argv: Optional[list[str]] = None) -> int: return 1 if not _selection_accepts_pr_diff(best, pr_base_commit, args.pr_head): return 1 - best.pop("_patch_input", None) print(json.dumps(best)) return 0 diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index 22a0f93511f5..c925359bdb11 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -395,12 +395,6 @@ def main(argv: Optional[list[str]] = None) -> int: "--coverage-db is. Its `drift` is what the freshness gate decides on; the " "rest is recorded. Absent or unreadable declines the tier.", ) - parser.add_argument( - "--coverage-input", - default=None, - help="Optional INPUT_JSON used only by the coverage tier. Tier 1 always uses the " - "positional PR input.", - ) parser.add_argument( "--coverage-max-drift", type=int, @@ -478,17 +472,9 @@ def main(argv: Optional[list[str]] = None) -> int: ) if not note: # the gate passed; a note here means it did not try: - coverage_pr = pr - if args.coverage_input: - coverage_diff_input = _load_pr_inputs(Path(args.coverage_input)) - coverage_pr = PRInputs( - changed_files=pr.changed_files, - diffs=coverage_diff_input.diffs, - post_merge=pr.post_merge, - ) db = open_db(args.coverage_db) tier, note = apply_coverage_tier( - coverage_pr, + pr, selector.pairs, selector.handled, stages, diff --git a/tests/unittest/scripts/test_cbts_coverage_artifact.py b/tests/unittest/scripts/test_cbts_coverage_artifact.py index f34a8016569b..0f977150d285 100644 --- a/tests/unittest/scripts/test_cbts_coverage_artifact.py +++ b/tests/unittest/scripts/test_cbts_coverage_artifact.py @@ -67,13 +67,9 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: _git(repo, "config", "user.email", "cbts@example.com") _git(repo, "config", "user.name", "CBTS Test") source = repo / "source.py" - source.write_text("first\nold\nlast\n") - _git(repo, "add", "source.py") - _git(repo, "commit", "-m", "older coverage revision") - older_db = _git(repo, "rev-parse", "HEAD") - source.write_text("first\nbase\nlast\n") - _git(repo, "commit", "-am", "base") + _git(repo, "add", "source.py") + _git(repo, "commit", "-m", "base") base = _git(repo, "rev-parse", "HEAD") _git(repo, "checkout", "-b", "pr") @@ -94,22 +90,12 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: _git(repo, "checkout", "pr") self.assertEqual( - artifact._patch_apply_status(base, head, clean_db, repo, str(repo)).status, - "clean", + artifact._patch_apply_status(base, head, clean_db, repo, str(repo)), "clean" ) self.assertEqual( - artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)).status, + artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)), "conflict", ) - cumulative = artifact._patch_apply_status( - base, head, older_db, repo, str(repo), db_drift_status="ahead" - ) - self.assertEqual(cumulative.status, "clean") - self.assertEqual(cumulative.diff_base_commit, older_db) - self.assertEqual(cumulative.changed_files, ["source.py"]) - assert cumulative.diffs is not None - self.assertIn("-old", cumulative.diffs["source.py"]) - self.assertIn("+pr", cumulative.diffs["source.py"]) def test_accepts_artifact_collected_at_pr_base(self) -> None: with ( @@ -179,11 +165,7 @@ def extract(tarball: Path, destination: Path) -> bool: with ( mock.patch.object(artifact, "merge_base", return_value="pr-base"), mock.patch.object(artifact, "select_tarball", return_value=selection) as select, - mock.patch.object( - artifact, - "_patch_apply_status", - return_value=artifact._PatchApplyResult("clean", "pr-base"), - ) as apply, + mock.patch.object(artifact, "_patch_apply_status", return_value="clean") as apply, mock.patch.object(artifact, "download", side_effect=download), mock.patch.object(artifact, "extract", side_effect=extract), ): @@ -192,9 +174,7 @@ def extract(tarball: Path, destination: Path) -> bool: self.assertIsNotNone(ready) assert ready is not None select.assert_called_once_with("pr-base") - apply.assert_called_once_with( - "pr-base", "pr-head", "coverage-commit", db_drift_status="ahead" - ) + apply.assert_called_once_with("pr-base", "pr-head", "coverage-commit") connection = sqlite3.connect(ready["path"]) try: tests = { From 5868a3911258fb495b956188b8bffcd04684b584 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:01:12 +0800 Subject: [PATCH 07/10] [TRTLLM-15885][infra] require internal coverage Git mirror Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- .../cbts/coverage_selection/SELECTION.md | 5 +- .../cbts/coverage_selection/artifact.py | 9 +- tests/unittest/scripts/test_cbts.py | 96 +++++---- .../scripts/test_cbts_coverage_artifact.py | 196 ------------------ 4 files changed, 62 insertions(+), 244 deletions(-) delete mode 100644 tests/unittest/scripts/test_cbts_coverage_artifact.py diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 18a093d0abcb..6c39d12b99c6 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -189,8 +189,9 @@ There is no local-git path for measuring lag. The CI checkout is `depth: 1, noTa single-SHA refspec (`trtllm_utils.checkoutSpec`), so the coverage revision is not available there. The conflict check creates a temporary repository and fetches the checked-out PR head locally. The base and DB revisions come from `CBTS_COVERAGE_GIT_REPO`, bound to the same authenticated -`default-llm-repo` internal mirror used by normal CI checkouts. The public GitHub repository is -only a local-command fallback. The temporary repository never changes the CI checkout or its index. +`default-llm-repo` internal mirror used by normal CI checkouts. A missing mirror setting fails the +check closed instead of falling back to the public Git repository. The temporary repository never +changes the CI checkout or its index. The compare API answers unless the revision has not reached the public mirror yet (404) or the token is missing (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index a2d57c38c901..7286e54ec594 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -69,7 +69,6 @@ _URM = "https://urm.nvidia.com/artifactory" _GITHUB_COMPARE = "https://api.github.com/repos/NVIDIA/TensorRT-LLM/compare" -_GITHUB_GIT_REPO = "https://github.com/NVIDIA/TensorRT-LLM.git" _JENKINS_BASE = "https://prod.blsm.nvidia.com/sw-tensorrt-top-1/job/LLM/job/main/job/L0_PostMerge" # Cover the 30-commit freshness window plus missing or unsuccessful builds. _MAX_PROBE = 50 @@ -244,7 +243,13 @@ def _patch_apply_status( ) -> _PatchApplyStatus: """Check a squashed PR commit against the DB revision.""" repo_source = repo_root.resolve() - main_repo = upstream_url or os.environ.get(COVERAGE_GIT_REPO_ENV) or _GITHUB_GIT_REPO + main_repo = upstream_url or os.environ.get(COVERAGE_GIT_REPO_ENV) + if not main_repo: + print( + f"[artifact] {COVERAGE_GIT_REPO_ENV} is required for the patch check", + file=sys.stderr, + ) + return "unknown" checked_out_head = _run_git(["rev-parse", "HEAD"], cwd=repo_source) if ( checked_out_head is None diff --git a/tests/unittest/scripts/test_cbts.py b/tests/unittest/scripts/test_cbts.py index 8df099b1ef85..b64a0dff9f15 100644 --- a/tests/unittest/scripts/test_cbts.py +++ b/tests/unittest/scripts/test_cbts.py @@ -22,6 +22,7 @@ import json import shutil import sqlite3 +import subprocess import sys import tempfile import types @@ -82,46 +83,57 @@ def _load_main() -> ModuleType: cbts_main = _load_main() +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + + class CoverageArtifactTest(unittest.TestCase): - def test_selects_closest_complete_ancestor_pair(self) -> None: - commits = {104: "newer", 102: "older-three", 101: "older-one"} - - def exists(url: str) -> bool: - if "/103/" in url: - return url.endswith("cbts_pystart_report_x86_64.tar.gz") - return "/100/" not in url - - relations = { - "newer": (1, "behind"), - "older-three": (3, "ahead"), - "older-one": (1, "ahead"), - } - with ( - mock.patch.object(artifact, "latest_build_number", return_value=104), - mock.patch.object(artifact, "_exists", side_effect=exists), - mock.patch.object( - artifact, "build_commit", side_effect=lambda build, _base: commits[build] - ), - mock.patch.object( - artifact, "drift", side_effect=lambda commit, _base: relations[commit] - ), - mock.patch.object(artifact, "compare_distance", return_value=7) as lag, - ): - selected = artifact.select_tarball( - "pr-base", artifact_base="coverage", jenkins_base="jenkins", max_probe=5 - ) + def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo = Path(temp_dir) + _git(repo, "init") + _git(repo, "config", "user.email", "cbts@example.com") + _git(repo, "config", "user.name", "CBTS Test") + source = repo / "source.py" + source.write_text("first\nbase\nlast\n") + _git(repo, "add", "source.py") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "pr") + source.write_text("first\npr\nlast\n") + _git(repo, "commit", "-am", "pr") + head = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "db-clean", base) + (repo / "other.py").write_text("coverage revision\n") + _git(repo, "add", "other.py") + _git(repo, "commit", "-m", "non-conflicting db") + clean_db = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "db-conflict", base) + source.write_text("first\ndb\nlast\n") + _git(repo, "commit", "-am", "conflicting db") + conflicting_db = _git(repo, "rev-parse", "HEAD") + _git(repo, "checkout", "pr") - self.assertIsNotNone(selected) - assert selected is not None - self.assertEqual(selected["build"], 101) - self.assertEqual(selected["commit"], "older-one") - self.assertEqual(selected["drift"], 1) - self.assertEqual(selected["drift_status"], "ahead") - self.assertEqual( - [url.rsplit("/", 1)[-1] for url in selected["urls"]], - list(artifact.ARCH_TARBALL_NAMES), - ) - lag.assert_called_once_with("older-one") + self.assertEqual( + artifact._patch_apply_status(base, head, clean_db, repo, str(repo)), "clean" + ) + self.assertEqual( + artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)), + "conflict", + ) + with mock.patch.dict(artifact.os.environ, {artifact.COVERAGE_GIT_REPO_ENV: ""}): + self.assertEqual( + artifact._patch_apply_status(base, head, clean_db, repo), "unknown" + ) def test_accepts_artifact_collected_at_pr_base(self) -> None: with ( @@ -191,6 +203,7 @@ def extract(tarball: Path, destination: Path) -> bool: with ( mock.patch.object(artifact, "merge_base", return_value="pr-base"), mock.patch.object(artifact, "select_tarball", return_value=selection) as select, + mock.patch.object(artifact, "_patch_apply_status", return_value="clean") as apply, mock.patch.object(artifact, "download", side_effect=download), mock.patch.object(artifact, "extract", side_effect=extract), ): @@ -199,6 +212,7 @@ def extract(tarball: Path, destination: Path) -> bool: self.assertIsNotNone(ready) assert ready is not None select.assert_called_once_with("pr-base") + apply.assert_called_once_with("pr-base", "pr-head", "coverage-commit") connection = sqlite3.connect(ready["path"]) try: tests = { @@ -215,12 +229,6 @@ def extract(tarball: Path, destination: Path) -> bool: ) self.assertEqual(json.loads(Path(ready["meta"]).read_text()), selection) - def test_freshness_gate_honors_configured_threshold(self) -> None: - self.assertEqual(cbts_main._coverage_freshness(7, 7), ("ok", "")) - freshness, reason = cbts_main._coverage_freshness(8, 7) - self.assertEqual(freshness, "stale") - self.assertTrue(reason) - # Coverage pilot diff --git a/tests/unittest/scripts/test_cbts_coverage_artifact.py b/tests/unittest/scripts/test_cbts_coverage_artifact.py deleted file mode 100644 index 0f977150d285..000000000000 --- a/tests/unittest/scripts/test_cbts_coverage_artifact.py +++ /dev/null @@ -1,196 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Tests for CBTS coverage artifact selection and architecture DB merging.""" - -from __future__ import annotations - -import importlib.util -import json -import shutil -import sqlite3 -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path -from types import ModuleType -from unittest import mock - -_ROOT = Path(__file__).resolve().parents[3] -_CBTS = _ROOT / "jenkins/scripts/cbts" -_ARTIFACT_PATH = _CBTS / "coverage_selection/artifact.py" -sys.path.insert(0, str(_CBTS / "coverage_utils")) - -from compact_db import write_leaf_database # noqa: E402 - - -def _load_artifact() -> ModuleType: - spec = importlib.util.spec_from_file_location("cbts_coverage_artifact", _ARTIFACT_PATH) - if spec is None or spec.loader is None: - raise RuntimeError(f"cannot load {_ARTIFACT_PATH}") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -artifact = _load_artifact() - - -def _git(repo: Path, *args: str) -> str: - return subprocess.run( - ["git", *args], - cwd=repo, - check=True, - stdout=subprocess.PIPE, - text=True, - ).stdout.strip() - - -class CoverageArtifactTest(unittest.TestCase): - def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - repo = Path(temp_dir) - _git(repo, "init") - _git(repo, "config", "user.email", "cbts@example.com") - _git(repo, "config", "user.name", "CBTS Test") - source = repo / "source.py" - source.write_text("first\nbase\nlast\n") - _git(repo, "add", "source.py") - _git(repo, "commit", "-m", "base") - base = _git(repo, "rev-parse", "HEAD") - - _git(repo, "checkout", "-b", "pr") - source.write_text("first\npr\nlast\n") - _git(repo, "commit", "-am", "pr") - head = _git(repo, "rev-parse", "HEAD") - - _git(repo, "checkout", "-b", "db-clean", base) - (repo / "other.py").write_text("coverage revision\n") - _git(repo, "add", "other.py") - _git(repo, "commit", "-m", "non-conflicting db") - clean_db = _git(repo, "rev-parse", "HEAD") - - _git(repo, "checkout", "-b", "db-conflict", base) - source.write_text("first\ndb\nlast\n") - _git(repo, "commit", "-am", "conflicting db") - conflicting_db = _git(repo, "rev-parse", "HEAD") - _git(repo, "checkout", "pr") - - self.assertEqual( - artifact._patch_apply_status(base, head, clean_db, repo, str(repo)), "clean" - ) - self.assertEqual( - artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)), - "conflict", - ) - - def test_accepts_artifact_collected_at_pr_base(self) -> None: - with ( - mock.patch.object(artifact, "latest_build_number", return_value=7), - mock.patch.object(artifact, "_exists", return_value=True), - mock.patch.object(artifact, "build_commit", return_value="pr-base"), - mock.patch.object(artifact, "drift", return_value=(0, "identical")), - mock.patch.object(artifact, "compare_distance", return_value=4), - ): - selected = artifact.select_tarball("pr-base", max_probe=1) - - self.assertIsNotNone(selected) - assert selected is not None - self.assertEqual(selected["drift"], 0) - self.assertEqual(selected["drift_status"], "identical") - - def test_prepare_merges_x86_and_sbsa_databases(self) -> None: - with tempfile.TemporaryDirectory() as temp_dir: - root = Path(temp_dir) - x86_db = root / "x86.sqlite" - sbsa_db = root / "sbsa.sqlite" - output_dir = root / "prepared" - write_leaf_database( - x86_db, - stage="A10-PyTorch-1", - process_uid="A10-PyTorch-1/coordinator", - touches={ - "A10-PyTorch-1/test_x86.py::test_one": { - ("/workspace/tensorrt_llm/x86.py", "run") - } - }, - outcomes={"A10-PyTorch-1/test_x86.py::test_one": "passed"}, - expected_workers={"A10-PyTorch-1/test_x86.py::test_one": 0}, - ) - write_leaf_database( - sbsa_db, - stage="GH200-PyTorch-1", - process_uid="GH200-PyTorch-1/coordinator", - touches={ - "GH200-PyTorch-1/test_sbsa.py::test_two": { - ("/workspace/tensorrt_llm/sbsa.py", "run") - } - }, - outcomes={"GH200-PyTorch-1/test_sbsa.py::test_two": "passed"}, - expected_workers={"GH200-PyTorch-1/test_sbsa.py::test_two": 0}, - ) - urls = artifact.tarball_urls(42) - selection = { - "url": urls[0], - "urls": urls, - "build": 42, - "commit": "coverage-commit", - "lag": 5, - "base_commit": "pr-base", - "drift": 2, - "drift_status": "ahead", - } - - def download(url: str, destination: Path) -> Path: - return destination / url.rsplit("/", 1)[-1] - - def extract(tarball: Path, destination: Path) -> bool: - source = x86_db if "x86_64" in tarball.name else sbsa_db - shutil.copyfile(source, destination / artifact.DB_NAME) - return True - - with ( - mock.patch.object(artifact, "merge_base", return_value="pr-base"), - mock.patch.object(artifact, "select_tarball", return_value=selection) as select, - mock.patch.object(artifact, "_patch_apply_status", return_value="clean") as apply, - mock.patch.object(artifact, "download", side_effect=download), - mock.patch.object(artifact, "extract", side_effect=extract), - ): - ready = artifact.prepare(str(output_dir), "pr-head") - - self.assertIsNotNone(ready) - assert ready is not None - select.assert_called_once_with("pr-base") - apply.assert_called_once_with("pr-base", "pr-head", "coverage-commit") - connection = sqlite3.connect(ready["path"]) - try: - tests = { - row[0] for row in connection.execute("SELECT DISTINCT test FROM touch_rows") - } - finally: - connection.close() - self.assertEqual( - tests, - { - "A10-PyTorch-1/test_x86.py::test_one", - "GH200-PyTorch-1/test_sbsa.py::test_two", - }, - ) - self.assertEqual(json.loads(Path(ready["meta"]).read_text()), selection) - - -if __name__ == "__main__": - unittest.main() From ea8f3097aebf9e43380eafe445ee1547740b4605 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:20:16 +0800 Subject: [PATCH 08/10] [TRTLLM-15885][test] bound CBTS git helper runtime Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- tests/unittest/scripts/test_cbts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unittest/scripts/test_cbts.py b/tests/unittest/scripts/test_cbts.py index b64a0dff9f15..d8892628d6bc 100644 --- a/tests/unittest/scripts/test_cbts.py +++ b/tests/unittest/scripts/test_cbts.py @@ -90,6 +90,7 @@ def _git(repo: Path, *args: str) -> str: check=True, stdout=subprocess.PIPE, text=True, + timeout=120, ).stdout.strip() From 6c54033f3930ac27e18bdbfd5e0a36c1f8c24421 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Wed, 16 Sep 2026 19:26:28 +0800 Subject: [PATCH 09/10] [TRTLLM-15885][infra] scope coverage compatibility to residual Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 51 ++++++--- jenkins/scripts/cbts/README.md | 5 +- .../cbts/coverage_selection/SELECTION.md | 46 +++++--- .../cbts/coverage_selection/artifact.py | 107 +++++++++++++++--- jenkins/scripts/cbts/coverage_tier.py | 25 +++- jenkins/scripts/cbts/main.py | 53 +++++++++ jenkins/scripts/cbts/tools/dryrun.py | 18 +++ .../cbts/tools/report_cbts_decision.py | 4 + tests/unittest/scripts/test_cbts.py | 49 +++++++- 9 files changed, 303 insertions(+), 55 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index cfc7ea28c871..5a4759a34d77 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -961,9 +961,6 @@ def getCbtsResult(pipeline, testFilter, globalVars) // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs. sh "apt-get update -qq && apt-get install -y -qq python3-yaml" - // Download the touch DB only for PRs in the coverage-tier pilot. - def coverageDb = _cbtsCoverageAudit(pipeline) - // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs", @@ -995,12 +992,32 @@ def getCbtsResult(pipeline, testFilter, globalVars) writeFile file: inputPath, text: inputJson def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json" - if (coverageDb) { - mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-meta ${coverageDb.meta}" - } def output = sh(script: mainCmd, returnStdout: true) - def result = _cbtsParseSelectionResult(output) + + // Tier 1 owns the definition of handled files. Only prepare a coverage DB when + // its residual is eligible for Tier 2, and scope the compatibility check to it. + if (result.scope == null && result.coverage_residual_files && + !result.coverage_decline_reason) { + def residualPath = "${LLM_ROOT}/cbts_coverage_residual.json" + writeFile file: residualPath, + text: groovy.json.JsonOutput.toJson(result.coverage_residual_files) + def coverageDb = _cbtsCoverageAudit(pipeline, residualPath) + if (coverageDb?.meta) { + def coverageCmd = mainCmd + " --coverage-db-meta ${coverageDb.meta}" + if (coverageDb.path) { + coverageCmd += " --coverage-db ${coverageDb.path}" + } + output = sh(script: coverageCmd, returnStdout: true) + result = _cbtsParseSelectionResult(output) + } else if (coverageDb?.compatibility) { + result.coverage_compatibility = coverageDb.compatibility + result.coverage_decline_reason = coverageDb.decline_reason ?: "" + result.coverage_decline_category = "coverage_unavailable" + output = groovy.json.JsonOutput.toJson(result) + } + } + if (result.scope == null) { pipeline.echo("CBTS: deferring — Python returned scope=null. " + "Reasons: ${result.reasons.join('; ')}") @@ -1061,9 +1078,9 @@ def _cbtsMultiGpuLabelGateOpen(pipeline, globalVars) } } -// Check pilot eligibility, then fetch and audit the touch DB; artifact.py's -// {path, meta} verbatim, or null on failure. -def _cbtsCoverageAudit(pipeline) +// Check pilot eligibility, then fetch and audit the touch DB. A compatibility +// decline returns metadata without a DB path so the reason remains observable. +def _cbtsCoverageAudit(pipeline, String residualPath) { try { // artifact.py resolves, downloads and merges the x86/SBSA DBs; paths come back @@ -1089,7 +1106,8 @@ def _cbtsCoverageAudit(pipeline) if (pilotEligible) { readyJson = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + - "--prepare cbts_cov${prHead ? " --pr-head ${prHead}" : ""} || true", + "--prepare cbts_cov --paths-json ${residualPath}" + + "${prHead ? " --pr-head ${prHead}" : ""} || true", returnStdout: true, ).trim() } @@ -1100,10 +1118,17 @@ def _cbtsCoverageAudit(pipeline) } if (!readyJson) { pipeline.echo("CBTS audit: no coverage DB could be prepared — skipping Tier 2") - return null + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB could not be prepared", + ] } def ready = new groovy.json.JsonSlurper().parseText(readyJson) - pipeline.echo("CBTS audit: PR diff applies cleanly to the latest coverage DB") + if (!ready.path) { + pipeline.echo("CBTS audit: Tier-2 residual conflicts with the latest coverage DB") + return ready + } + pipeline.echo("CBTS audit: Tier-2 residual applies cleanly to the latest coverage DB") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}" return ready } catch (InterruptedException e) { diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index eb39f3440b65..ca1fe81132d3 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -325,8 +325,9 @@ CBTS defers to the existing filter chain when: decorator line), has no usable patch, has unparsable source, or has a closure change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs -- The current PR diff cannot be applied without conflicts to the latest complete coverage DB - revision (or the check cannot be completed) — Tier 2 declines before download +- The Tier-2 residual cannot be applied without conflicts to the latest complete coverage DB + revision (or the check cannot be completed) — Tier 1-owned file conflicts are ignored, while + residual conflicts decline Tier 2 before download - The resolved DB sits more than `--coverage-max-drift` commits from the PR's base commit, on either side, or an unmeasurable distance from it — Tier 2 declines (`coverage_freshness` = `stale` / `unknown`) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 6c39d12b99c6..410ca8c88eaf 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -17,7 +17,9 @@ Tier 2 coverage residual is all core Python and all present in the DB → sco full fallback scope=null ``` -Tier 2 only ever looks at the **residual**: the files no Tier 1 rule claimed. +Tier 2 only ever looks at the **residual**: the files no Tier 1 rule claimed. The Git +compatibility gate uses that same residual. Conflicts in files already claimed by Tier 1, such as +`waives.txt`, do not disable Tier 2. ## 2. The qualname concepts @@ -156,6 +158,7 @@ stages finish, it uploads `cbts_pystart_report_x86_64.tar.gz` and ### 8.1 Resolution ``` +run Tier 1 → Tier-2 residual paths GitHub compare main... → PR base commit Jenkins REST lastBuild → newest build number N for b in N .. N-49: (_MAX_PROBE) @@ -166,8 +169,8 @@ for b in N .. N-49: (_MAX_PROBE) fetch PR head locally; fetch base and latest DB from the normal CI Git mirror create a squashed PR commit with `commit-tree`, parented at the PR base cherry-pick it onto the DB revision - clean → continue - conflict / unavailable revision → decline Tier 2 + no unmerged residual path → continue + residual conflict / unavailable revision → decline Tier 2 ``` Requiring the pair prevents the selector from narrowing only one CPU architecture. Builds are @@ -199,7 +202,7 @@ routinely already spent). An unmeasurable candidate-to-base relation is rejected candidate's main-tip lag may remain `null`. The token comes from the `github-cred-trtllm-ci` credential — the one `getGithubMRChangedFile` -already uses — bound around the `--print-selection` call in `_cbtsCoverageAudit` and read from +already uses — bound around the artifact selection call in `_cbtsCoverageAudit` and read from `GITHUB_API_TOKEN`. This number reports overall freshness. It does not select the DB. @@ -212,25 +215,28 @@ freshness gate and telemetry. `ahead`, `behind`, and `identical` describe valid diverged and unknown relations decline Tier 2. `commit-tree` represents the complete base-to-head PR change as one commit, and `cherry-pick` -tests that commit against the latest DB without serializing through patch format. This check is -independent of whether the DB is older than, equal to, or newer than the PR base. A conflict or an -unmeasurable check declines before the large DB artifacts are downloaded. If it passes, Tier 2 uses -the original forge PR payload. `--coverage-max-drift` applies a second fail-closed bound: beyond 30 -commits Tier 2 declines and the PR runs in full. +tests that commit against the latest DB without serializing through patch format. When the +cherry-pick reports conflicts, only unmerged paths in the Tier-2 residual count; conflicts in +Tier-1-owned files are ignored. This check is independent of whether the DB is older than, equal +to, or newer than the PR base. A residual conflict or an unmeasurable check declines before the +large DB artifacts are downloaded. If it passes, Tier 2 uses the original forge PR payload. +`--coverage-max-drift` applies a second fail-closed bound: beyond 30 commits Tier 2 declines and +the PR runs in full. ### 8.3 What happens with the result -`--prepare DIR` does the whole fetch in one call: resolve the PR base, select the latest complete -pair, validate compatibility, stream both tarballs down, unpack their identically named SQLite -files separately, and union them with `compact_db.merge_databases`. It writes the selection JSON +After Tier 1 computes the residual, `--prepare DIR --paths-json PATH` resolves the PR base, selects +the latest complete pair, validates residual compatibility, streams both tarballs down, unpacks +their identically named SQLite files separately, and unions them with `compact_db.merge_databases`. +It writes the selection JSON beside the merged SQLite as `cbts_coverage_db.json` and prints `{path, meta}`. Groovy binds the credentials, logs the successful compatibility check, and runs `coverage_audit.py` over the result. -Any failure anywhere is caught and non-fatal: no prepared DB is returned, Tier 2 never runs, and the -PR gets a full run. +On a residual conflict it returns `{path: null, meta}` so the decline remains observable without +downloading the DB. Any failure is non-fatal: Tier 2 never runs and the PR gets a full run. -Those two paths reach `main.py` as `--coverage-db` and `--coverage-db-meta`, so a new selection -field needs no Groovy change. `main.py` records all of it and **gates on the drift**: past +The metadata path always reaches `main.py`; the DB path is added only when compatibility is clean. +`main.py` records all of it and **gates on the drift**: past `--coverage-max-drift` (default 30) the tier declines and the PR runs in full, on the grounds that a DB that far from the PR's base no longer describes who touches what in the code under test. A drift that could not be measured — including a meta file that is missing or unreadable — is @@ -247,6 +253,10 @@ All of it lands in the decision and in OpenSearch: | `coverage_db_drift` | `l_coverage_db_drift` | **the gated number**; `null` / `-1` when unmeasurable | | `coverage_db_drift_status` | `s_coverage_db_drift_status` | `ahead` / `behind` / `identical` for every selected DB | | `coverage_freshness` | `s_coverage_freshness` | `ok` / `stale` / `unknown`, empty when no DB | +| `coverage_compatibility` | `s_coverage_compatibility` | `clean` / `conflict` / `unknown` / `not_attempted` | +| `coverage_decline_reason` | `s_coverage_decline_reason` | human-readable Tier-2 decline detail | +| `coverage_decline_category` | `s_coverage_decline_category` | aggregation-safe decline category | +| count of `coverage_residual_files` | `l_coverage_residual_files` | Tier-2 opportunity size | so the decline rate is queryable per verdict rather than only readable in `s_reason`. @@ -267,6 +277,10 @@ so the decline rate is queryable per verdict rather than only readable in `s_rea "coverage_db_base_commit": "9f0da65d...", "coverage_db_drift": 7, "coverage_db_drift_status": "ahead", + "coverage_residual_files": ["tensorrt_llm/example.py"], + "coverage_compatibility": "clean", + "coverage_decline_reason": "", + "coverage_decline_category": "", "coverage_no_diff_files": 0, "reasons": [{"source": "coverage", "impacted": 118, "untrusted": 104, ...}] } diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index 7286e54ec594..d190101124ce 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -15,7 +15,8 @@ Candidates are recent builds of `` with both x86 and SBSA coverage tarballs. The newest complete pair wins. A squashed PR commit must -cherry-pick cleanly to that DB's revision; otherwise Tier 2 declines. Revision +cherry-pick cleanly for Tier 2's residual paths at that DB's revision; +otherwise Tier 2 declines. Revision metadata comes from the forge compare API and needs `GITHUB_API_TOKEN`, the anonymous quota being per-IP and exhausted by shared CI egress. @@ -240,8 +241,9 @@ def _patch_apply_status( db_commit: str, repo_root: Path = Path("."), upstream_url: Optional[str] = None, + relevant_paths: Optional[list[str]] = None, ) -> _PatchApplyStatus: - """Check a squashed PR commit against the DB revision.""" + """Check the relevant paths of a squashed PR commit against the DB revision.""" repo_source = repo_root.resolve() main_repo = upstream_url or os.environ.get(COVERAGE_GIT_REPO_ENV) if not main_repo: @@ -317,18 +319,54 @@ def _patch_apply_status( applied = _run_git(["cherry-pick", "--no-commit", pr_commit], cwd=repo) if applied is None: return "unknown" - return "clean" if applied.returncode == 0 else "conflict" + if applied.returncode == 0: + return "clean" + if relevant_paths is None: + return "conflict" + unmerged = _run_git( + ["diff", "--name-only", "--diff-filter=U", "-z"], + cwd=repo, + ) + if unmerged is None or unmerged.returncode != 0: + return "unknown" + conflict_paths = { + path.decode("utf-8", "surrogateescape") for path in unmerged.stdout.split(b"\0") if path + } + if not conflict_paths: + return "unknown" + relevant_conflicts = conflict_paths.intersection(relevant_paths) + if relevant_conflicts: + return "conflict" + print( + "[artifact] ignoring conflict(s) outside the Tier-2 residual: " + + ", ".join(sorted(conflict_paths)), + file=sys.stderr, + ) + return "clean" -def _selection_accepts_pr_diff(sel: dict, pr_base_commit: str, pr_head: str) -> bool: - sel["patch_apply_status"] = _patch_apply_status(pr_base_commit, pr_head, sel["commit"]) +def _selection_accepts_pr_diff( + sel: dict, + pr_base_commit: str, + pr_head: str, + relevant_paths: Optional[list[str]] = None, +) -> bool: + sel["patch_apply_status"] = _patch_apply_status( + pr_base_commit, + pr_head, + sel["commit"], + relevant_paths=relevant_paths, + ) if sel["patch_apply_status"] == "clean": + sel["coverage_decline_reason"] = "" + sel["coverage_decline_category"] = "" return True - print( - f"[artifact] coverage tier declined: PR diff does not apply cleanly to " - f"latest DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})", - file=sys.stderr, + sel["coverage_decline_reason"] = ( + "coverage tier declined: Tier-2 residual does not apply cleanly to " + f"latest DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})" ) + sel["coverage_decline_category"] = f"compatibility_{sel['patch_apply_status']}" + print(f"[artifact] {sel['coverage_decline_reason']}", file=sys.stderr) return False @@ -450,7 +488,11 @@ def extract(tarball: Path, dest: Path) -> bool: return True -def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: +def prepare( + dest_dir: str, + pr_head: Optional[str], + relevant_paths: list[str], +) -> Optional[dict]: """Resolve, download, and merge the DB pair; `{path, meta}` or None on failure. `meta` is the selection JSON on disk, which `main.py --coverage-db-meta` reads. @@ -466,12 +508,21 @@ def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: sel = select_tarball(pr_base_commit) if sel is None: return None - if not _selection_accepts_pr_diff(sel, pr_base_commit, pr_head): - return None + compatible = _selection_accepts_pr_diff( + sel, + pr_base_commit, + pr_head, + relevant_paths, + ) print(describe(sel), file=sys.stderr) dest = Path(dest_dir) dest.mkdir(parents=True, exist_ok=True) + meta = dest / META_NAME + meta.write_text(json.dumps(sel)) + if not compatible: + return {"path": None, "meta": str(meta)} + db = dest / DB_NAME with tempfile.TemporaryDirectory(prefix="cbts_artifacts_", dir=dest) as temp_dir: temp = Path(temp_dir) @@ -494,11 +545,24 @@ def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: return None connection.close() - meta = dest / META_NAME - meta.write_text(json.dumps(sel)) return {"path": str(db), "meta": str(meta)} +def _load_paths_json(path: Optional[str]) -> Optional[list[str]]: + """Load the non-empty JSON list of repository-relative Tier-2 paths.""" + if not path: + return None + try: + data = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"[artifact] residual paths unreadable ({path}): {e}", file=sys.stderr) + return None + if not isinstance(data, list) or not data or not all(isinstance(item, str) for item in data): + print("[artifact] residual paths must be a non-empty JSON string list", file=sys.stderr) + return None + return sorted(set(data)) + + def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -525,13 +589,21 @@ def main(argv: Optional[list[str]] = None) -> int: help="required PR head revision; its merge base measures DB drift and defines " "the diff checked against the latest coverage revision", ) + ap.add_argument( + "--paths-json", + default=None, + help="JSON list of Tier-2 residual paths; required for the compatibility check", + ) args = ap.parse_args(argv) if not args.print_selection and not args.prepare: ap.error("one of --print-selection / --prepare is required") if args.prepare: - ready = prepare(args.prepare, args.pr_head) + relevant_paths = _load_paths_json(args.paths_json) + if relevant_paths is None: + return 1 + ready = prepare(args.prepare, args.pr_head, relevant_paths) if ready is None: return 1 print(json.dumps(ready)) @@ -540,6 +612,9 @@ def main(argv: Optional[list[str]] = None) -> int: if not args.pr_head: print("[artifact] --pr-head is required", file=sys.stderr) return 1 + relevant_paths = _load_paths_json(args.paths_json) + if relevant_paths is None: + return 1 pr_base_commit = merge_base(args.pr_head) if not pr_base_commit: return 1 @@ -563,7 +638,7 @@ def main(argv: Optional[list[str]] = None) -> int: best = select_tarball(pr_base_commit) if best is None: return 1 - if not _selection_accepts_pr_diff(best, pr_base_commit, args.pr_head): + if not _selection_accepts_pr_diff(best, pr_base_commit, args.pr_head, relevant_paths): return 1 print(json.dumps(best)) return 0 diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index bd647fd21105..2b6a89fdca9b 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -171,6 +171,23 @@ def _build_narrowing( return removed, dropped, must_run +def coverage_preflight( + pr: PRInputs, + pairs: list[tuple[object, RuleResult]], + handled: set[str], +) -> tuple[list[str], str]: + """Return the Tier-2 residual and any reason it cannot be audited.""" + residual = sorted(set(pr.changed_files) - handled) + if any(result.scope is None for _, result in pairs): + return residual, "coverage tier skipped: a rule forced fallback (scope=null)" + if not residual: + return residual, "coverage tier skipped: no residual (all files handled by rules)" + for path in residual: + if not (path.endswith(".py") and path.startswith("tensorrt_llm/")): + return residual, f"coverage tier declined: non-core-Python residual file: {path}" + return residual, "" + + def apply_coverage_tier( pr: PRInputs, pairs: list[tuple[object, RuleResult]], @@ -182,11 +199,9 @@ def apply_coverage_tier( no_data_policy: str = DEFAULT_NO_DATA_POLICY, ) -> tuple[CoverageTierResult | None, str]: """Return (narrowing, note); narrowing is None when the tier keeps the Tier-1 result.""" - if any(r.scope is None for _, r in pairs): - return None, "coverage tier skipped: a rule forced fallback (scope=null)" - residual = sorted(set(pr.changed_files) - handled) - if not residual: - return None, "coverage tier skipped: no residual (all files handled by rules)" + residual, note = coverage_preflight(pr, pairs, handled) + if note: + return None, note selector = CoverageSelector(db, repo_root, no_data_policy=no_data_policy) cov = selector.decide(residual, pr.diffs) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index c925359bdb11..83a7b2c7e08f 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -55,6 +55,7 @@ NO_DATA_POLICIES, apply_coverage_tier, compute_coverage_stage_counts, + coverage_preflight, open_db, write_coverage_test_db, ) @@ -146,6 +147,13 @@ class SelectionResult: coverage_db_drift_status: str = "" # Freshness verdict on that drift: ok / stale / unknown; empty when no DB was consulted. coverage_freshness: str = "" + # Files left after Tier 1; the patch compatibility check is scoped to these paths. + coverage_residual_files: list[str] = field(default_factory=list) + # clean / conflict / unknown / not_attempted. + coverage_compatibility: str = "not_attempted" + # Stable classification for coverage-tier fallback telemetry. + coverage_decline_reason: str = "" + coverage_decline_category: str = "" # Residual files the forge API returned no patch for; they fall back to file level. coverage_no_diff_files: int = 0 @@ -169,6 +177,10 @@ def to_json(self) -> str: "coverage_db_drift": self.coverage_db_drift, "coverage_db_drift_status": self.coverage_db_drift_status, "coverage_freshness": self.coverage_freshness, + "coverage_residual_files": list(self.coverage_residual_files), + "coverage_compatibility": self.coverage_compatibility, + "coverage_decline_reason": self.coverage_decline_reason, + "coverage_decline_category": self.coverage_decline_category, "coverage_no_diff_files": self.coverage_no_diff_files, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -202,6 +214,28 @@ def _coverage_freshness(drift: Optional[int], max_drift: int) -> tuple[str, str] return "ok", "" +def _coverage_decline_category(reason: str) -> str: + """Map a human-readable Tier-2 decline reason to a stable telemetry category.""" + categories = ( + ("a rule forced fallback", "rule_forced_fallback"), + ("no residual", "no_residual"), + ("non-core-Python residual file", "non_core_python"), + ("freshness unknown", "freshness_unknown"), + ("over the", "freshness_stale"), + ("zero-touch residual file", "zero_touch"), + ("no usable diff", "no_usable_diff"), + ("import-executed change", "import_executed"), + ("unparsable source", "unparsable_source"), + ("closure change", "closure_change"), + ("coverage tier errored", "tier_error"), + ) + return ( + next((category for text, category in categories if text in reason), "other") + if reason + else "" + ) + + def _rule_reason(rule, r) -> dict: """One rule's structured reason entry: `{source, blocks, stages}`.""" return { @@ -456,6 +490,13 @@ def main(argv: Optional[list[str]] = None) -> int: rules = build_rules(yaml_index, stages, repo_root) selector = Selector(stages) result = selector.run(pr, rules) + if result.scope is None: + result.coverage_residual_files, result.coverage_decline_reason = coverage_preflight( + pr, selector.pairs, selector.handled + ) + result.coverage_decline_category = _coverage_decline_category( + result.coverage_decline_reason + ) meta = _load_coverage_db_meta(args.coverage_db_meta) result.coverage_db_build = meta.get("build") @@ -464,12 +505,22 @@ def main(argv: Optional[list[str]] = None) -> int: result.coverage_db_drift = meta.get("drift") result.coverage_db_base_commit = meta.get("base_commit") result.coverage_db_drift_status = meta.get("drift_status") or "" + result.coverage_compatibility = meta.get("patch_apply_status") or "not_attempted" + result.coverage_decline_reason = ( + meta.get("coverage_decline_reason") or result.coverage_decline_reason + ) + result.coverage_decline_category = ( + meta.get("coverage_decline_category") or result.coverage_decline_category + ) if args.coverage_db and result.scope is None: tier = None result.coverage_freshness, note = _coverage_freshness( result.coverage_db_drift, args.coverage_max_drift ) + if note: + result.coverage_decline_reason = note + result.coverage_decline_category = _coverage_decline_category(note) if not note: # the gate passed; a note here means it did not try: db = open_db(args.coverage_db) @@ -519,6 +570,8 @@ def main(argv: Optional[list[str]] = None) -> int: cov_reason ] elif note: + result.coverage_decline_reason = note + result.coverage_decline_category = _coverage_decline_category(note) for x in result.reasons: if isinstance(x, dict) and x.get("source") == "fallback": x["coverage_declined"] = note diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index 68467e533a91..cc37fc95ed75 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -203,6 +203,12 @@ def _fmt_summary( lines.append(f"scopes: {result.get('scopes', [])}") lines.append(f"sanity_required: {result.get('sanity_required')}") lines.append(f"perfsanity_required: {result.get('perfsanity_required')}") + lines.append(f"coverage_compatibility: {result.get('coverage_compatibility')}") + lines.append(f"coverage_decline_category: {result.get('coverage_decline_category')!r}") + lines.append(f"coverage_decline_reason: {result.get('coverage_decline_reason')!r}") + residual = result.get("coverage_residual_files", []) + lines.append(f"coverage_residual_files ({len(residual)}):") + lines.extend(f" - {path}" for path in residual) override = result.get("test_db_dir_override") lines.append(f"test_db_dir_override: {override!r}") stages = result.get("affected_stages", []) @@ -373,9 +379,15 @@ def _write_index( range_expr: Optional[str] = None, ) -> None: counts: dict[Optional[str], int] = {} + compatibility_counts: dict[str, int] = {} + decline_counts: dict[str, int] = {} for _, _, _, _, result, _ in rows: scope = result.get("scope") if "_error" not in result else "ERROR" counts[scope] = counts.get(scope, 0) + 1 + compatibility = result.get("coverage_compatibility") or "not_attempted" + compatibility_counts[compatibility] = compatibility_counts.get(compatibility, 0) + 1 + category = result.get("coverage_decline_category") or "none" + decline_counts[category] = decline_counts.get(category, 0) + 1 trigger = "/bot run --post-merge" if post_merge else "/bot run" if range_expr: @@ -411,6 +423,12 @@ def _sort_key(row): lines += ["", "## Scope distribution", ""] for scope, n in sorted(counts.items(), key=lambda kv: -kv[1]): lines.append(f"- {scope or 'None (fallback)'}: {n}") + lines += ["", "## Coverage compatibility distribution", ""] + for compatibility, n in sorted(compatibility_counts.items(), key=lambda kv: -kv[1]): + lines.append(f"- {compatibility}: {n}") + lines += ["", "## Coverage decline distribution", ""] + for category, n in sorted(decline_counts.items(), key=lambda kv: -kv[1]): + lines.append(f"- {category}: {n}") (out_dir / "INDEX.md").write_text("\n".join(lines) + "\n") diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index 5970d801efa0..f4f093c3bd77 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -230,6 +230,10 @@ def build_document( "s_coverage_db_drift_status": decision.get("coverage_db_drift_status") or "", # Freshness-gate verdict on that drift: ok / stale / unknown; empty when no DB was consulted. "s_coverage_freshness": decision.get("coverage_freshness") or "", + "s_coverage_compatibility": decision.get("coverage_compatibility") or "not_attempted", + "s_coverage_decline_reason": decision.get("coverage_decline_reason") or "", + "s_coverage_decline_category": decision.get("coverage_decline_category") or "", + "l_coverage_residual_files": len(decision.get("coverage_residual_files") or []), # This field is consumed directly by the CBTS OpenSearch dashboard. "d_case_skip_rate": round(case_skip_rate, 4), "b_case_skip_rate_valid": case_skip_rate_valid, diff --git a/tests/unittest/scripts/test_cbts.py b/tests/unittest/scripts/test_cbts.py index d8892628d6bc..922cefc72167 100644 --- a/tests/unittest/scripts/test_cbts.py +++ b/tests/unittest/scripts/test_cbts.py @@ -102,13 +102,16 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: _git(repo, "config", "user.email", "cbts@example.com") _git(repo, "config", "user.name", "CBTS Test") source = repo / "source.py" + waives = repo / "waives.txt" source.write_text("first\nbase\nlast\n") - _git(repo, "add", "source.py") + waives.write_text("base\n") + _git(repo, "add", "source.py", "waives.txt") _git(repo, "commit", "-m", "base") base = _git(repo, "rev-parse", "HEAD") _git(repo, "checkout", "-b", "pr") source.write_text("first\npr\nlast\n") + waives.write_text("pr\n") _git(repo, "commit", "-am", "pr") head = _git(repo, "rev-parse", "HEAD") @@ -122,6 +125,11 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: source.write_text("first\ndb\nlast\n") _git(repo, "commit", "-am", "conflicting db") conflicting_db = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "db-irrelevant-conflict", base) + waives.write_text("db\n") + _git(repo, "commit", "-am", "conflicting non-residual file") + irrelevant_conflict_db = _git(repo, "rev-parse", "HEAD") _git(repo, "checkout", "pr") self.assertEqual( @@ -131,6 +139,28 @@ def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)), "conflict", ) + self.assertEqual( + artifact._patch_apply_status( + base, + head, + irrelevant_conflict_db, + repo, + str(repo), + ["source.py"], + ), + "clean", + ) + self.assertEqual( + artifact._patch_apply_status( + base, + head, + irrelevant_conflict_db, + repo, + str(repo), + ["waives.txt"], + ), + "conflict", + ) with mock.patch.dict(artifact.os.environ, {artifact.COVERAGE_GIT_REPO_ENV: ""}): self.assertEqual( artifact._patch_apply_status(base, head, clean_db, repo), "unknown" @@ -208,12 +238,17 @@ def extract(tarball: Path, destination: Path) -> bool: mock.patch.object(artifact, "download", side_effect=download), mock.patch.object(artifact, "extract", side_effect=extract), ): - ready = artifact.prepare(str(output_dir), "pr-head") + ready = artifact.prepare(str(output_dir), "pr-head", ["tensorrt_llm/source.py"]) self.assertIsNotNone(ready) assert ready is not None select.assert_called_once_with("pr-base") - apply.assert_called_once_with("pr-base", "pr-head", "coverage-commit") + apply.assert_called_once_with( + "pr-base", + "pr-head", + "coverage-commit", + relevant_paths=["tensorrt_llm/source.py"], + ) connection = sqlite3.connect(ready["path"]) try: tests = { @@ -509,6 +544,10 @@ def test_build_document_filters_unscheduled_stages_and_persists_valid_rate( "H100-PyTorch-1": 1, "H100-4_GPUs-PyTorch-1": 1, } + decision["coverage_compatibility"] = "conflict" + decision["coverage_decline_reason"] = "residual conflict" + decision["coverage_decline_category"] = "compatibility_conflict" + decision["coverage_residual_files"] = ["tensorrt_llm/source.py"] document = report_module.build_document( decision, @@ -525,6 +564,10 @@ def test_build_document_filters_unscheduled_stages_and_persists_valid_rate( assert document["b_case_skip_rate_valid"] is True assert document["b_non_cbts_multi_gpu_required"] is True assert document["b_multi_gpu_label_gate_open"] is False + assert document["s_coverage_compatibility"] == "conflict" + assert document["s_coverage_decline_reason"] == "residual conflict" + assert document["s_coverage_decline_category"] == "compatibility_conflict" + assert document["l_coverage_residual_files"] == 1 assert document["flat_detail"]["hit_stages"] == [ "H100-PyTorch-1", "H100-PyTorch-2", From a486564bcb57af5ec3a23f048efababf5216b351 Mon Sep 17 00:00:00 2001 From: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:31:32 +0800 Subject: [PATCH 10/10] [None][infra] pin CBTS coverage DB per PR head Signed-off-by: Ivy Zhang <25222398+crazydemo@users.noreply.github.com> --- jenkins/L0_MergeRequest.groovy | 176 +++++++++++++++--- jenkins/scripts/cbts/README.md | 3 +- .../cbts/coverage_selection/SELECTION.md | 48 +++-- .../cbts/coverage_selection/artifact.py | 130 +++++++++---- tests/unittest/scripts/test_cbts.py | 85 +++++++++ 5 files changed, 359 insertions(+), 83 deletions(-) diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index 5a4759a34d77..5aee03761503 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -157,6 +157,12 @@ def INFRA_DRY_RUN = "infra_dry_run" // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. @Field def ENABLE_CBTS_COVERAGE = true +@Field +def CBTS_COVERAGE_PIN_VERSION = 1 +@Field +def CBTS_COVERAGE_PIN_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/cbts/coverage-db-pins/v1" +@Field +def URM_ARTIFACTORY_BASE = "https://urm.nvidia.com/artifactory" // Version-controlled Tier 2 rollout policy. Keep this in the infra-owned Groovy // boundary so changing who receives coverage-based narrowing requires infra review. @Field @@ -1002,7 +1008,7 @@ def getCbtsResult(pipeline, testFilter, globalVars) def residualPath = "${LLM_ROOT}/cbts_coverage_residual.json" writeFile file: residualPath, text: groovy.json.JsonOutput.toJson(result.coverage_residual_files) - def coverageDb = _cbtsCoverageAudit(pipeline, residualPath) + def coverageDb = _cbtsCoverageAudit(pipeline, globalVars, residualPath) if (coverageDb?.meta) { def coverageCmd = mainCmd + " --coverage-db-meta ${coverageDb.meta}" if (coverageDb.path) { @@ -1080,7 +1086,7 @@ def _cbtsMultiGpuLabelGateOpen(pipeline, globalVars) // Check pilot eligibility, then fetch and audit the touch DB. A compatibility // decline returns metadata without a DB path so the reason remains observable. -def _cbtsCoverageAudit(pipeline, String residualPath) +def _cbtsCoverageAudit(pipeline, globalVars, String residualPath) { try { // artifact.py resolves, downloads and merges the x86/SBSA DBs; paths come back @@ -1090,12 +1096,10 @@ def _cbtsCoverageAudit(pipeline, String residualPath) def readyJson = "" def prAuthor = "" def pilotEligible = false - withCredentials([ - usernamePassword( - credentialsId: 'github-cred-trtllm-ci', - usernameVariable: 'NOT_USED_YET', - passwordVariable: 'GITHUB_API_TOKEN'), - string(credentialsId: 'default-llm-repo', variable: 'CBTS_COVERAGE_GIT_REPO'), + withCredentials([usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN'), ]) { prAuthor = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_pilot.py", @@ -1103,18 +1107,132 @@ def _cbtsCoverageAudit(pipeline, String residualPath) ).trim() pilotEligible = prAuthor && CBTS_COVERAGE_PILOT_USERS.any { it.equalsIgnoreCase(prAuthor) } pipeline.echo("CBTS coverage pilot: pr_author=${prAuthor ?: 'unknown'}, eligible=${pilotEligible}") - if (pilotEligible) { - readyJson = sh( + } + if (!pilotEligible) { + pipeline.echo("CBTS: coverage tier disabled for this PR — running Tier 1 only") + return null + } + + def prNumber = _cbtsPrNumber(globalVars) + if (!prHead || !prNumber) { + pipeline.echo("CBTS audit: PR number/head unavailable; cannot make coverage DB selection repeatable") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: PR identity unavailable for coverage DB pin", + ] + } + + def pinPath = "${LLM_ROOT}/cbts_db_pin.json" + def pinUrl = "${URM_ARTIFACTORY_BASE}/${CBTS_COVERAGE_PIN_BASE}/${prNumber}/${prHead}/cbts_db_pin.json" + if (fileExists(pinPath)) { + deleteFile pinPath + } + def pinStatus = sh( + script: "curl --noproxy '*' -sS -o ${pinPath} -w '%{http_code}' " + + "--connect-timeout 5 --max-time 30 '${pinUrl}' || true", + returnStdout: true, + ).trim() + def pinnedBuild = null + def pinnedCommit = "" + def pinExists = pinStatus == "200" + if (pinExists) { + def pin = new groovy.json.JsonSlurper().parseText(readFile(file: pinPath)) + def validPin = pin instanceof Map && + pin.version?.toString() == CBTS_COVERAGE_PIN_VERSION.toString() && + pin.pr_number?.toString() == prNumber.toString() && + pin.pr_head?.toString() == prHead && + pin.coverage_db_build?.toString() ==~ /[1-9]\d*/ && + pin.coverage_db_commit?.toString() ==~ /[0-9a-fA-F]{40}/ + if (!validPin) { + pipeline.echo("CBTS audit: coverage DB pin is malformed or does not match this PR head") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: invalid coverage DB pin", + ] + } + pinnedBuild = pin.coverage_db_build.toString().toInteger() + pinnedCommit = pin.coverage_db_commit.toString() + pipeline.echo("CBTS audit: reusing pinned coverage DB build ${pinnedBuild}") + } else if (pinStatus == "404") { + pipeline.echo("CBTS audit: no pin for this PR head; selecting the latest coverage DB") + } else { + pipeline.echo("CBTS audit: coverage DB pin query failed (HTTP ${pinStatus ?: 'unknown'})") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB pin query failed", + ] + } + + if (!pinExists) { + def selectionJson = "" + withCredentials([usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN'), + ]) { + selectionJson = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + - "--prepare cbts_cov --paths-json ${residualPath}" + - "${prHead ? " --pr-head ${prHead}" : ""} || true", + "--resolve-build --pr-head ${prHead} || true", returnStdout: true, ).trim() } + if (!selectionJson) { + pipeline.echo("CBTS audit: latest coverage DB could not be resolved; pin not written") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB could not be resolved", + ] + } + def selected = new groovy.json.JsonSlurper().parseText(selectionJson) + if (!(selected instanceof Map) || + !(selected.build?.toString() ==~ /[1-9]\d*/) || + !(selected.commit?.toString() ==~ /[0-9a-fA-F]{40}/)) { + pipeline.echo("CBTS audit: selected coverage DB metadata is invalid; pin not written") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: selected coverage DB metadata invalid", + ] + } + def pin = [ + version: CBTS_COVERAGE_PIN_VERSION, + pr_number: prNumber.toString(), + pr_head: prHead, + coverage_db_build: selected.build, + coverage_db_commit: selected.commit, + ] + writeFile file: pinPath, text: groovy.json.JsonOutput.toJson(pin) + try { + def pinTarget = "${CBTS_COVERAGE_PIN_BASE}/${prNumber}/${prHead}/" + trtllm_utils.uploadArtifacts(pinPath, pinTarget) + pinnedBuild = selected.build.toString().toInteger() + pinnedCommit = selected.commit.toString() + pipeline.echo("CBTS audit: pinned coverage DB build ${pinnedBuild} for PR head ${prHead}") + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("CBTS audit: coverage DB pin upload failed (${e.message})") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB pin upload failed", + ] + } } - if (!pilotEligible) { - pipeline.echo("CBTS: coverage tier disabled for this PR — running Tier 1 only") - return null + + withCredentials([ + usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN'), + string(credentialsId: 'default-llm-repo', variable: 'CBTS_COVERAGE_GIT_REPO'), + ]) { + readyJson = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + + "--prepare cbts_cov --paths-json ${residualPath}" + + " --pr-head ${prHead}" + + " --build ${pinnedBuild}" + + " --expected-commit ${pinnedCommit} || true", + returnStdout: true, + ).trim() } if (!readyJson) { pipeline.echo("CBTS audit: no coverage DB could be prepared — skipping Tier 2") @@ -1125,10 +1243,10 @@ def _cbtsCoverageAudit(pipeline, String residualPath) } def ready = new groovy.json.JsonSlurper().parseText(readyJson) if (!ready.path) { - pipeline.echo("CBTS audit: Tier-2 residual conflicts with the latest coverage DB") + pipeline.echo("CBTS audit: Tier-2 residual conflicts with the selected coverage DB") return ready } - pipeline.echo("CBTS audit: Tier-2 residual applies cleanly to the latest coverage DB") + pipeline.echo("CBTS audit: Tier-2 residual applies cleanly to the selected coverage DB") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}" return ready } catch (InterruptedException e) { @@ -1139,6 +1257,19 @@ def _cbtsCoverageAudit(pipeline, String residualPath) } } +def _cbtsPrNumber(globalVars) +{ + def prUrl = globalVars[GITHUB_PR_API_URL] + if (prUrl) { + def match = (prUrl =~ /\/pulls?\/(\d+)/) + if (match.find()) { + return match.group(1) + } + return "" + } + return env.gitlabMergeRequestIid ?: "" +} + // Post one CBTS decision record to OpenSearch (best-effort; never blocks CI). // decisionJson null for deferred; reason used only then. Context/creds via env. // Multi-GPU enters the pre-merge denominator only when normal CI requires it @@ -1163,16 +1294,7 @@ def _cbtsReportDecision(pipeline, globalVars, String status, String reason, Stri // PR number for s_pr_number, mirroring perf_regression_utils: GitHub PR // builds carry it in github_pr_api_url (.../pulls/); GitLab MR builds // expose env.gitlabMergeRequestIid. Empty for post-merge/branch builds. - def prNumber = "" - def prUrl = globalVars[GITHUB_PR_API_URL] - if (prUrl) { - def m = (prUrl =~ /\/pulls?\/(\d+)/) - if (m.find()) { - prNumber = m.group(1) - } - } else { - prNumber = env.gitlabMergeRequestIid ?: "" - } + def prNumber = _cbtsPrNumber(globalVars) if (prNumber) { args += " --pr-number ${prNumber}" } diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index ca1fe81132d3..a7769471dc2b 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -325,7 +325,8 @@ CBTS defers to the existing filter chain when: decorator line), has no usable patch, has unparsable source, or has a closure change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs -- The Tier-2 residual cannot be applied without conflicts to the latest complete coverage DB +- The Tier-2 residual cannot be applied without conflicts to the selected coverage DB (the + freshest complete build on a PR head's first run, then pinned for that head) revision (or the check cannot be completed) — Tier 1-owned file conflicts are ignored, while residual conflicts decline Tier 2 before download - The resolved DB sits more than `--coverage-max-drift` commits from the PR's diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 410ca8c88eaf..7d592cb3fb6a 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -160,23 +160,38 @@ stages finish, it uploads `cbts_pystart_report_x86_64.tar.gz` and ``` run Tier 1 → Tier-2 residual paths GitHub compare main... → PR base commit -Jenkins REST lastBuild → newest build number N -for b in N .. N-49: (_MAX_PROBE) - ranged GET both architecture tarballs → skip b unless both exist - GET build_info.txt, parse `commit=` → sha; skip when absent - first complete pair with a known sha → latest coverage DB - compare ... → record topology and absolute distance -fetch PR head locally; fetch base and latest DB from the normal CI Git mirror +GET Artifactory pin at /: + 200 → reuse its coverage build + 404 → resolve the newest build below + other / malformed → decline Tier 2 +when no pin: + Jenkins REST lastBuild → newest build number N + for b in N .. N-49: (_MAX_PROBE) + ranged GET both architecture tarballs → skip b unless both exist + GET build_info.txt, parse `commit=` → sha; skip when absent + first complete pair with a known sha → latest coverage DB + upload the build/commit pin → stable for this PR head +compare ... → record topology and absolute distance +fetch PR head locally; fetch base and selected DB from the normal CI Git mirror create a squashed PR commit with `commit-tree`, parented at the PR base cherry-pick it onto the DB revision no unmerged residual path → continue residual conflict / unavailable revision → decline Tier 2 ``` -Requiring the pair prevents the selector from narrowing only one CPU architecture. Builds are -probed newest first, and the first complete pair with commit metadata is the only DB considered. -The selector does not substitute an older DB merely because it is closer to the PR base or because -the latest DB conflicts with the PR diff. +The pin lives under +`LLM/main/cbts/coverage-db-pins/v1///cbts_db_pin.json`. A new PR head has no +pin and therefore receives the freshest complete pair available at its first CBTS run. Repeated +`/bot run` commands for the same head reuse that build even after newer post-merge DBs appear. +Pin lookup, validation, and first-write upload fail closed; CBTS never silently substitutes a +newer build when a pin cannot be read or written. Preparation also verifies that the pinned +build's `build_info.txt` still names the commit recorded in the pin, so an overwritten or +corrupted build cannot silently change a repeated run. + +Requiring the architecture pair prevents the selector from narrowing only one CPU architecture. +For an unpinned head, builds are probed newest first and the first complete pair with commit +metadata is the only DB considered. The selector does not substitute an older DB merely because +it is closer to the PR base or because the selected DB conflicts with the PR diff. ### 8.2 Measuring the lag (reporting) @@ -209,13 +224,13 @@ This number reports overall freshness. It does not select the DB. ### 8.2b Measuring the drift (gating) -The PR base is `merge_base_commit.sha` from `main...`. The latest DB is compared as +The PR base is `merge_base_commit.sha` from `main...`. The selected DB is compared as `...`, and drift is `ahead_by + behind_by`: an absolute distance used only by the freshness gate and telemetry. `ahead`, `behind`, and `identical` describe valid positions on main; diverged and unknown relations decline Tier 2. `commit-tree` represents the complete base-to-head PR change as one commit, and `cherry-pick` -tests that commit against the latest DB without serializing through patch format. When the +tests that commit against the selected DB without serializing through patch format. When the cherry-pick reports conflicts, only unmerged paths in the Tier-2 residual count; conflicts in Tier-1-owned files are ignored. This check is independent of whether the DB is older than, equal to, or newer than the PR base. A residual conflict or an unmeasurable check declines before the @@ -226,9 +241,10 @@ the PR runs in full. ### 8.3 What happens with the result -After Tier 1 computes the residual, `--prepare DIR --paths-json PATH` resolves the PR base, selects -the latest complete pair, validates residual compatibility, streams both tarballs down, unpacks -their identically named SQLite files separately, and unions them with `compact_db.merge_databases`. +After Tier 1 computes the residual, Groovy resolves and persists a build pin before any large +download. `--prepare DIR --build BUILD --paths-json PATH` then resolves the PR base, validates the +pinned pair's residual compatibility, streams both tarballs down, unpacks their identically named +SQLite files separately, and unions them with `compact_db.merge_databases`. It writes the selection JSON beside the merged SQLite as `cbts_coverage_db.json` and prints `{path, meta}`. Groovy binds the credentials, logs the successful compatibility check, and runs `coverage_audit.py` over the result. diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index d190101124ce..7af39690dad4 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -23,9 +23,11 @@ The selected architecture DBs are merged locally through the compact coverage schema, producing the one DB consumed by `main.py`. -Two entry points. `--print-selection` prints the selection metadata and stops. -`--prepare DIR` goes on to download, unpack, and merge the winner, drop that -JSON beside it, and print the two paths consumed by CBTS. +Three entry points. `--resolve-build` prints the newest (or explicitly pinned) +build metadata without checking or downloading the PR diff. `--print-selection` +also checks diff compatibility and stops. `--prepare DIR` downloads, unpacks, +and merges the winner, drops that JSON beside it, and prints the two paths +consumed by CBTS. """ from __future__ import annotations @@ -363,7 +365,7 @@ def _selection_accepts_pr_diff( return True sel["coverage_decline_reason"] = ( "coverage tier declined: Tier-2 residual does not apply cleanly to " - f"latest DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})" + f"selected DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})" ) sel["coverage_decline_category"] = f"compatibility_{sel['patch_apply_status']}" print(f"[artifact] {sel['coverage_decline_reason']}", file=sys.stderr) @@ -423,19 +425,50 @@ def select_tarball( return None -def measure_drift(sel: dict, pr_head: Optional[str]) -> dict: - """Add PR-base relation metadata to a pinned selection, in place.""" - sel.setdefault("base_commit", None) - sel.setdefault("drift", None) - sel.setdefault("drift_status", "unknown") - if not pr_head or not sel.get("commit"): - return sel - base = merge_base(pr_head) - if not base: - return sel - sel["base_commit"] = base - sel["drift"], sel["drift_status"] = drift(sel["commit"], base) - return sel +def select_build( + build: int, + pr_base_commit: str, + expected_commit: Optional[str] = None, + artifact_base: str = ARTIFACT_BASE, +) -> Optional[dict]: + """Resolve and validate one explicitly pinned coverage build.""" + urls = tarball_urls(build, artifact_base) + if not all(_exists(url) for url in urls): + print(f"[artifact] pinned build {build}: coverage pair is incomplete", file=sys.stderr) + return None + commit = build_commit(build, artifact_base) + if not commit: + print(f"[artifact] pinned build {build}: commit unknown", file=sys.stderr) + return None + if expected_commit is not None and commit != expected_commit: + print( + f"[artifact] pinned build {build}: commit changed from {expected_commit} to {commit}", + file=sys.stderr, + ) + return None + distance, status = drift(commit, pr_base_commit) + if distance is None: + print( + f"[artifact] pinned build {build}: topology against the PR base unknown", + file=sys.stderr, + ) + return None + if status not in ("ahead", "behind", "identical"): + print( + f"[artifact] pinned build {build}: relation to the PR base is {status or 'unknown'}", + file=sys.stderr, + ) + return None + return { + "url": urls[0], + "urls": urls, + "build": build, + "commit": commit, + "base_commit": pr_base_commit, + "drift": distance, + "drift_status": status, + "lag": compare_distance(commit), + } def describe(sel: dict) -> str: @@ -492,6 +525,8 @@ def prepare( dest_dir: str, pr_head: Optional[str], relevant_paths: list[str], + build: Optional[int] = None, + expected_commit: Optional[str] = None, ) -> Optional[dict]: """Resolve, download, and merge the DB pair; `{path, meta}` or None on failure. @@ -505,7 +540,11 @@ def prepare( if not pr_base_commit: print("[artifact] could not resolve the PR base commit", file=sys.stderr) return None - sel = select_tarball(pr_base_commit) + sel = ( + select_build(build, pr_base_commit, expected_commit=expected_commit) + if build is not None + else select_tarball(pr_base_commit) + ) if sel is None: return None compatible = _selection_accepts_pr_diff( @@ -567,6 +606,11 @@ def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) + ap.add_argument( + "--resolve-build", + action="store_true", + help="resolve and print build metadata without checking or downloading the PR diff", + ) ap.add_argument( "--print-selection", action="store_true", @@ -576,6 +620,11 @@ def main(argv: Optional[list[str]] = None) -> int: ap.add_argument( "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" ) + ap.add_argument( + "--expected-commit", + default=None, + help="require an explicitly pinned build to retain this coverage commit", + ) ap.add_argument( "--prepare", metavar="DIR", @@ -587,7 +636,7 @@ def main(argv: Optional[list[str]] = None) -> int: "--pr-head", default=None, help="required PR head revision; its merge base measures DB drift and defines " - "the diff checked against the latest coverage revision", + "the diff checked against the selected coverage revision", ) ap.add_argument( "--paths-json", @@ -596,14 +645,22 @@ def main(argv: Optional[list[str]] = None) -> int: ) args = ap.parse_args(argv) - if not args.print_selection and not args.prepare: - ap.error("one of --print-selection / --prepare is required") + if sum((args.resolve_build, args.print_selection, args.prepare is not None)) != 1: + ap.error("exactly one of --resolve-build / --print-selection / --prepare is required") + if args.expected_commit is not None and args.build is None: + ap.error("--expected-commit requires --build") if args.prepare: relevant_paths = _load_paths_json(args.paths_json) if relevant_paths is None: return 1 - ready = prepare(args.prepare, args.pr_head, relevant_paths) + ready = prepare( + args.prepare, + args.pr_head, + relevant_paths, + build=args.build, + expected_commit=args.expected_commit, + ) if ready is None: return 1 print(json.dumps(ready)) @@ -612,32 +669,27 @@ def main(argv: Optional[list[str]] = None) -> int: if not args.pr_head: print("[artifact] --pr-head is required", file=sys.stderr) return 1 - relevant_paths = _load_paths_json(args.paths_json) - if relevant_paths is None: - return 1 pr_base_commit = merge_base(args.pr_head) if not pr_base_commit: return 1 if args.build is not None: - urls = tarball_urls(args.build) - if not all(_exists(url) for url in urls): - return 1 - commit = build_commit(args.build) - best = { - "url": urls[0], - "urls": urls, - "build": args.build, - "commit": commit, - "lag": compare_distance(commit) if commit else None, - } - measure_drift(best, args.pr_head) - if best["drift"] is None or best["drift_status"] not in ("ahead", "behind", "identical"): - return 1 + best = select_build( + args.build, + pr_base_commit, + expected_commit=args.expected_commit, + ) else: best = select_tarball(pr_base_commit) if best is None: return 1 + if args.resolve_build: + print(json.dumps(best)) + return 0 + + relevant_paths = _load_paths_json(args.paths_json) + if relevant_paths is None: + return 1 if not _selection_accepts_pr_diff(best, pr_base_commit, args.pr_head, relevant_paths): return 1 print(json.dumps(best)) diff --git a/tests/unittest/scripts/test_cbts.py b/tests/unittest/scripts/test_cbts.py index 922cefc72167..5738cf3cbf4f 100644 --- a/tests/unittest/scripts/test_cbts.py +++ b/tests/unittest/scripts/test_cbts.py @@ -19,6 +19,7 @@ import ast import importlib.util +import io import json import shutil import sqlite3 @@ -181,6 +182,90 @@ def test_accepts_artifact_collected_at_pr_base(self) -> None: self.assertEqual(selected["drift"], 0) self.assertEqual(selected["drift_status"], "identical") + def test_select_build_resolves_explicit_pinned_build(self) -> None: + with ( + mock.patch.object(artifact, "_exists", return_value=True), + mock.patch.object(artifact, "build_commit", return_value="coverage-commit"), + mock.patch.object(artifact, "drift", return_value=(3, "behind")), + mock.patch.object(artifact, "compare_distance", return_value=7), + ): + selected = artifact.select_build(42, "pr-base") + + self.assertIsNotNone(selected) + assert selected is not None + self.assertEqual(selected["build"], 42) + self.assertEqual(selected["commit"], "coverage-commit") + self.assertEqual(selected["base_commit"], "pr-base") + self.assertEqual(selected["drift"], 3) + + def test_select_build_rejects_changed_pinned_commit(self) -> None: + with ( + mock.patch.object(artifact, "_exists", return_value=True), + mock.patch.object(artifact, "build_commit", return_value="replacement-commit"), + mock.patch.object(artifact, "drift") as drift, + ): + selected = artifact.select_build( + 42, + "pr-base", + expected_commit="pinned-commit", + ) + + self.assertIsNone(selected) + drift.assert_not_called() + + def test_resolve_build_prints_metadata_without_residual_paths(self) -> None: + selection = { + "build": 42, + "commit": "coverage-commit", + "base_commit": "pr-base", + } + stdout = io.StringIO() + with ( + mock.patch.object(artifact, "merge_base", return_value="pr-base"), + mock.patch.object(artifact, "select_tarball", return_value=selection), + mock.patch("sys.stdout", stdout), + ): + status = artifact.main(["--resolve-build", "--pr-head", "pr-head"]) + + self.assertEqual(status, 0) + self.assertEqual(json.loads(stdout.getvalue()), selection) + + def test_prepare_uses_explicit_pinned_build(self) -> None: + selection = { + "url": "x86-url", + "urls": ["x86-url", "sbsa-url"], + "build": 42, + "commit": "coverage-commit", + "base_commit": "pr-base", + "drift": 2, + "drift_status": "behind", + "lag": 5, + } + with ( + tempfile.TemporaryDirectory() as temp_dir, + mock.patch.object(artifact, "merge_base", return_value="pr-base"), + mock.patch.object(artifact, "select_build", return_value=selection) as select_build, + mock.patch.object(artifact, "select_tarball") as select_latest, + mock.patch.object(artifact, "_patch_apply_status", return_value="conflict"), + ): + ready = artifact.prepare( + temp_dir, + "pr-head", + ["tensorrt_llm/source.py"], + build=42, + expected_commit="coverage-commit", + ) + + self.assertIsNotNone(ready) + assert ready is not None + self.assertIsNone(ready["path"]) + select_build.assert_called_once_with( + 42, + "pr-base", + expected_commit="coverage-commit", + ) + select_latest.assert_not_called() + def test_prepare_merges_x86_and_sbsa_databases(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir)