diff --git a/jenkins/L0_MergeRequest.groovy b/jenkins/L0_MergeRequest.groovy index c8670231c391..5aee03761503 100644 --- a/jenkins/L0_MergeRequest.groovy +++ b/jenkins/L0_MergeRequest.groovy @@ -157,6 +157,12 @@ def INFRA_DRY_RUN = "infra_dry_run" // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1. @Field def ENABLE_CBTS_COVERAGE = true +@Field +def CBTS_COVERAGE_PIN_VERSION = 1 +@Field +def CBTS_COVERAGE_PIN_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/cbts/coverage-db-pins/v1" +@Field +def URM_ARTIFACTORY_BASE = "https://urm.nvidia.com/artifactory" // Version-controlled Tier 2 rollout policy. Keep this in the infra-owned Groovy // boundary so changing who receives coverage-based narrowing requires infra review. @Field @@ -961,9 +967,6 @@ def getCbtsResult(pipeline, testFilter, globalVars) // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs. sh "apt-get update -qq && apt-get install -y -qq python3-yaml" - // Download the touch DB only for PRs in the coverage-tier pilot. - def coverageDb = _cbtsCoverageAudit(pipeline) - // Ask Python which file patterns need diffs, fetch them. def patternsOut = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs", @@ -995,12 +998,32 @@ def getCbtsResult(pipeline, testFilter, globalVars) writeFile file: inputPath, text: inputJson def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json" - if (coverageDb) { - mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-meta ${coverageDb.meta}" - } def output = sh(script: mainCmd, returnStdout: true) - def result = _cbtsParseSelectionResult(output) + + // Tier 1 owns the definition of handled files. Only prepare a coverage DB when + // its residual is eligible for Tier 2, and scope the compatibility check to it. + if (result.scope == null && result.coverage_residual_files && + !result.coverage_decline_reason) { + def residualPath = "${LLM_ROOT}/cbts_coverage_residual.json" + writeFile file: residualPath, + text: groovy.json.JsonOutput.toJson(result.coverage_residual_files) + def coverageDb = _cbtsCoverageAudit(pipeline, globalVars, residualPath) + if (coverageDb?.meta) { + def coverageCmd = mainCmd + " --coverage-db-meta ${coverageDb.meta}" + if (coverageDb.path) { + coverageCmd += " --coverage-db ${coverageDb.path}" + } + output = sh(script: coverageCmd, returnStdout: true) + result = _cbtsParseSelectionResult(output) + } else if (coverageDb?.compatibility) { + result.coverage_compatibility = coverageDb.compatibility + result.coverage_decline_reason = coverageDb.decline_reason ?: "" + result.coverage_decline_category = "coverage_unavailable" + output = groovy.json.JsonOutput.toJson(result) + } + } + if (result.scope == null) { pipeline.echo("CBTS: deferring — Python returned scope=null. " + "Reasons: ${result.reasons.join('; ')}") @@ -1061,9 +1084,9 @@ def _cbtsMultiGpuLabelGateOpen(pipeline, globalVars) } } -// Check pilot eligibility, then fetch and audit the touch DB; artifact.py's -// {path, meta} verbatim, or null on failure. -def _cbtsCoverageAudit(pipeline) +// Check pilot eligibility, then fetch and audit the touch DB. A compatibility +// decline returns metadata without a DB path so the reason remains observable. +def _cbtsCoverageAudit(pipeline, globalVars, String residualPath) { try { // artifact.py resolves, downloads and merges the x86/SBSA DBs; paths come back @@ -1073,30 +1096,157 @@ def _cbtsCoverageAudit(pipeline) def readyJson = "" def prAuthor = "" def pilotEligible = false - withCredentials([usernamePassword(credentialsId: 'github-cred-trtllm-ci', usernameVariable: 'NOT_USED_YET', passwordVariable: 'GITHUB_API_TOKEN')]) { + withCredentials([usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN'), + ]) { prAuthor = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_pilot.py", returnStdout: true, ).trim() pilotEligible = prAuthor && CBTS_COVERAGE_PILOT_USERS.any { it.equalsIgnoreCase(prAuthor) } pipeline.echo("CBTS coverage pilot: pr_author=${prAuthor ?: 'unknown'}, eligible=${pilotEligible}") - if (pilotEligible) { - readyJson = sh( + } + if (!pilotEligible) { + pipeline.echo("CBTS: coverage tier disabled for this PR — running Tier 1 only") + return null + } + + def prNumber = _cbtsPrNumber(globalVars) + if (!prHead || !prNumber) { + pipeline.echo("CBTS audit: PR number/head unavailable; cannot make coverage DB selection repeatable") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: PR identity unavailable for coverage DB pin", + ] + } + + def pinPath = "${LLM_ROOT}/cbts_db_pin.json" + def pinUrl = "${URM_ARTIFACTORY_BASE}/${CBTS_COVERAGE_PIN_BASE}/${prNumber}/${prHead}/cbts_db_pin.json" + if (fileExists(pinPath)) { + deleteFile pinPath + } + def pinStatus = sh( + script: "curl --noproxy '*' -sS -o ${pinPath} -w '%{http_code}' " + + "--connect-timeout 5 --max-time 30 '${pinUrl}' || true", + returnStdout: true, + ).trim() + def pinnedBuild = null + def pinnedCommit = "" + def pinExists = pinStatus == "200" + if (pinExists) { + def pin = new groovy.json.JsonSlurper().parseText(readFile(file: pinPath)) + def validPin = pin instanceof Map && + pin.version?.toString() == CBTS_COVERAGE_PIN_VERSION.toString() && + pin.pr_number?.toString() == prNumber.toString() && + pin.pr_head?.toString() == prHead && + pin.coverage_db_build?.toString() ==~ /[1-9]\d*/ && + pin.coverage_db_commit?.toString() ==~ /[0-9a-fA-F]{40}/ + if (!validPin) { + pipeline.echo("CBTS audit: coverage DB pin is malformed or does not match this PR head") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: invalid coverage DB pin", + ] + } + pinnedBuild = pin.coverage_db_build.toString().toInteger() + pinnedCommit = pin.coverage_db_commit.toString() + pipeline.echo("CBTS audit: reusing pinned coverage DB build ${pinnedBuild}") + } else if (pinStatus == "404") { + pipeline.echo("CBTS audit: no pin for this PR head; selecting the latest coverage DB") + } else { + pipeline.echo("CBTS audit: coverage DB pin query failed (HTTP ${pinStatus ?: 'unknown'})") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB pin query failed", + ] + } + + if (!pinExists) { + def selectionJson = "" + withCredentials([usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN'), + ]) { + selectionJson = sh( script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + - "--prepare cbts_cov${prHead ? " --pr-head ${prHead}" : ""} || true", + "--resolve-build --pr-head ${prHead} || true", returnStdout: true, ).trim() } + if (!selectionJson) { + pipeline.echo("CBTS audit: latest coverage DB could not be resolved; pin not written") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB could not be resolved", + ] + } + def selected = new groovy.json.JsonSlurper().parseText(selectionJson) + if (!(selected instanceof Map) || + !(selected.build?.toString() ==~ /[1-9]\d*/) || + !(selected.commit?.toString() ==~ /[0-9a-fA-F]{40}/)) { + pipeline.echo("CBTS audit: selected coverage DB metadata is invalid; pin not written") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: selected coverage DB metadata invalid", + ] + } + def pin = [ + version: CBTS_COVERAGE_PIN_VERSION, + pr_number: prNumber.toString(), + pr_head: prHead, + coverage_db_build: selected.build, + coverage_db_commit: selected.commit, + ] + writeFile file: pinPath, text: groovy.json.JsonOutput.toJson(pin) + try { + def pinTarget = "${CBTS_COVERAGE_PIN_BASE}/${prNumber}/${prHead}/" + trtllm_utils.uploadArtifacts(pinPath, pinTarget) + pinnedBuild = selected.build.toString().toInteger() + pinnedCommit = selected.commit.toString() + pipeline.echo("CBTS audit: pinned coverage DB build ${pinnedBuild} for PR head ${prHead}") + } catch (InterruptedException e) { + throw e + } catch (Exception e) { + pipeline.echo("CBTS audit: coverage DB pin upload failed (${e.message})") + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB pin upload failed", + ] + } } - if (!pilotEligible) { - pipeline.echo("CBTS: coverage tier disabled for this PR — running Tier 1 only") - return null + + withCredentials([ + usernamePassword( + credentialsId: 'github-cred-trtllm-ci', + usernameVariable: 'NOT_USED_YET', + passwordVariable: 'GITHUB_API_TOKEN'), + string(credentialsId: 'default-llm-repo', variable: 'CBTS_COVERAGE_GIT_REPO'), + ]) { + readyJson = sh( + script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " + + "--prepare cbts_cov --paths-json ${residualPath}" + + " --pr-head ${prHead}" + + " --build ${pinnedBuild}" + + " --expected-commit ${pinnedCommit} || true", + returnStdout: true, + ).trim() } if (!readyJson) { pipeline.echo("CBTS audit: no coverage DB could be prepared — skipping Tier 2") - return null + return [ + compatibility: "unknown", + decline_reason: "coverage tier declined: coverage DB could not be prepared", + ] } def ready = new groovy.json.JsonSlurper().parseText(readyJson) + if (!ready.path) { + pipeline.echo("CBTS audit: Tier-2 residual conflicts with the selected coverage DB") + return ready + } + pipeline.echo("CBTS audit: Tier-2 residual applies cleanly to the selected coverage DB") sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}" return ready } catch (InterruptedException e) { @@ -1107,6 +1257,19 @@ def _cbtsCoverageAudit(pipeline) } } +def _cbtsPrNumber(globalVars) +{ + def prUrl = globalVars[GITHUB_PR_API_URL] + if (prUrl) { + def match = (prUrl =~ /\/pulls?\/(\d+)/) + if (match.find()) { + return match.group(1) + } + return "" + } + return env.gitlabMergeRequestIid ?: "" +} + // Post one CBTS decision record to OpenSearch (best-effort; never blocks CI). // decisionJson null for deferred; reason used only then. Context/creds via env. // Multi-GPU enters the pre-merge denominator only when normal CI requires it @@ -1131,16 +1294,7 @@ def _cbtsReportDecision(pipeline, globalVars, String status, String reason, Stri // PR number for s_pr_number, mirroring perf_regression_utils: GitHub PR // builds carry it in github_pr_api_url (.../pulls/); GitLab MR builds // expose env.gitlabMergeRequestIid. Empty for post-merge/branch builds. - def prNumber = "" - def prUrl = globalVars[GITHUB_PR_API_URL] - if (prUrl) { - def m = (prUrl =~ /\/pulls?\/(\d+)/) - if (m.find()) { - prNumber = m.group(1) - } - } else { - prNumber = env.gitlabMergeRequestIid ?: "" - } + def prNumber = _cbtsPrNumber(globalVars) if (prNumber) { args += " --pr-number ${prNumber}" } diff --git a/jenkins/scripts/cbts/README.md b/jenkins/scripts/cbts/README.md index 60251c871d65..a7769471dc2b 100644 --- a/jenkins/scripts/cbts/README.md +++ b/jenkins/scripts/cbts/README.md @@ -325,6 +325,10 @@ CBTS defers to the existing filter chain when: decorator line), has no usable patch, has unparsable source, or has a closure change with no wider row set (see `coverage_selection/SELECTION.md` §3-4) - No touch DB artifact could be resolved — Tier 2 never runs +- The Tier-2 residual cannot be applied without conflicts to the selected coverage DB (the + freshest complete build on a PR head's first run, then pinned for that head) + revision (or the check cannot be completed) — Tier 1-owned file conflicts are ignored, while + residual conflicts decline Tier 2 before download - The resolved DB sits more than `--coverage-max-drift` commits from the PR's base commit, on either side, or an unmeasurable distance from it — Tier 2 declines (`coverage_freshness` = `stale` / `unknown`) diff --git a/jenkins/scripts/cbts/coverage_selection/SELECTION.md b/jenkins/scripts/cbts/coverage_selection/SELECTION.md index 64188f38c506..7d592cb3fb6a 100644 --- a/jenkins/scripts/cbts/coverage_selection/SELECTION.md +++ b/jenkins/scripts/cbts/coverage_selection/SELECTION.md @@ -17,7 +17,9 @@ Tier 2 coverage residual is all core Python and all present in the DB → sco full fallback scope=null ``` -Tier 2 only ever looks at the **residual**: the files no Tier 1 rule claimed. +Tier 2 only ever looks at the **residual**: the files no Tier 1 rule claimed. The Git +compatibility gate uses that same residual. Conflicts in files already claimed by Tier 1, such as +`waives.txt`, do not disable Tier 2. ## 2. The qualname concepts @@ -156,20 +158,40 @@ stages finish, it uploads `cbts_pystart_report_x86_64.tar.gz` and ### 8.1 Resolution ``` +run Tier 1 → Tier-2 residual paths GitHub compare main... → PR base commit -Jenkins REST lastBuild → newest build number N -for b in N .. N-49: (_MAX_PROBE) - ranged GET both architecture tarballs → skip b unless both exist - GET build_info.txt, parse `commit=` → sha; skip when absent - compare ... → retain ancestors and exact matches - retain distance to the PR base -rank by (distance ascending, build descending) +GET Artifactory pin at /: + 200 → reuse its coverage build + 404 → resolve the newest build below + other / malformed → decline Tier 2 +when no pin: + Jenkins REST lastBuild → newest build number N + for b in N .. N-49: (_MAX_PROBE) + ranged GET both architecture tarballs → skip b unless both exist + GET build_info.txt, parse `commit=` → sha; skip when absent + first complete pair with a known sha → latest coverage DB + upload the build/commit pin → stable for this PR head +compare ... → record topology and absolute distance +fetch PR head locally; fetch base and selected DB from the normal CI Git mirror +create a squashed PR commit with `commit-tree`, parented at the PR base +cherry-pick it onto the DB revision + no unmerged residual path → continue + residual conflict / unavailable revision → decline Tier 2 ``` -Requiring the pair prevents the selector from narrowing only one CPU architecture. Ranking is by -**revision, not build number**: a post-merge build can be a re-run of an older commit, so the -highest build number is not necessarily the closest safe revision. Build number only breaks ties. -An exact PR-base match is allowed and wins with distance zero. +The pin lives under +`LLM/main/cbts/coverage-db-pins/v1///cbts_db_pin.json`. A new PR head has no +pin and therefore receives the freshest complete pair available at its first CBTS run. Repeated +`/bot run` commands for the same head reuse that build even after newer post-merge DBs appear. +Pin lookup, validation, and first-write upload fail closed; CBTS never silently substitutes a +newer build when a pin cannot be read or written. Preparation also verifies that the pinned +build's `build_info.txt` still names the commit recorded in the pin, so an overwritten or +corrupted build cannot silently change a repeated run. + +Requiring the architecture pair prevents the selector from narrowing only one CPU architecture. +For an unpinned head, builds are probed newest first and the first complete pair with commit +metadata is the only DB considered. The selector does not substitute an older DB merely because +it is closer to the PR base or because the selected DB conflicts with the PR diff. ### 8.2 Measuring the lag (reporting) @@ -181,10 +203,13 @@ Since every candidate revision is a commit that already merged to `main`, it can *behind* the tip: `behind_by` stays 0 and the lag is non-negative. A non-zero `behind_by` would mean the revision is no longer on `main` at all (history rewritten). -There is no local-git path. The CI checkout is `depth: 1, noTags: true` with a single-SHA refspec -(`trtllm_utils.checkoutSpec`), so no candidate revision is ever in the object store; a git -measurement would also answer against whatever ref it was given, and a merely stale ref returns a -*smaller* number rather than an error. +There is no local-git path for measuring lag. The CI checkout is `depth: 1, noTags: true` with a +single-SHA refspec (`trtllm_utils.checkoutSpec`), so the coverage revision is not available there. +The conflict check creates a temporary repository and fetches the checked-out PR head locally. +The base and DB revisions come from `CBTS_COVERAGE_GIT_REPO`, bound to the same authenticated +`default-llm-repo` internal mirror used by normal CI checkouts. A missing mirror setting fails the +check closed instead of falling back to the public Git repository. The temporary repository never +changes the CI checkout or its index. The compare API answers unless the revision has not reached the public mirror yet (404) or the token is missing (403 — the 60/h anonymous quota is shared across NVIDIA's egress IP and is @@ -192,34 +217,42 @@ routinely already spent). An unmeasurable candidate-to-base relation is rejected candidate's main-tip lag may remain `null`. The token comes from the `github-cred-trtllm-ci` credential — the one `getGithubMRChangedFile` -already uses — bound around the `--print-selection` call in `_cbtsCoverageAudit` and read from +already uses — bound around the artifact selection call in `_cbtsCoverageAudit` and read from `GITHUB_API_TOKEN`. -This number reports overall freshness. It is **not** what the gate decides on. +This number reports overall freshness. It does not select the DB. ### 8.2b Measuring the drift (gating) -The PR base is `merge_base_commit.sha` from `main...`. Each candidate is compared as -`...` and is eligible only when GitHub reports `ahead` (the PR base is ahead of the -DB) or `identical`. `behind`, `diverged`, and unknown relations are rejected before download, so a -coverage DB is never newer than the PR base. +The PR base is `merge_base_commit.sha` from `main...`. The selected DB is compared as +`...`, and drift is `ahead_by + behind_by`: an absolute distance used only by the +freshness gate and telemetry. `ahead`, `behind`, and `identical` describe valid positions on main; +diverged and unknown relations decline Tier 2. -For eligible candidates, drift is their plain ancestor distance to the PR base. The closest one -wins, and `--coverage-max-drift` applies a second fail-closed bound: beyond 30 commits Tier 2 -declines and the PR runs in full. +`commit-tree` represents the complete base-to-head PR change as one commit, and `cherry-pick` +tests that commit against the selected DB without serializing through patch format. When the +cherry-pick reports conflicts, only unmerged paths in the Tier-2 residual count; conflicts in +Tier-1-owned files are ignored. This check is independent of whether the DB is older than, equal +to, or newer than the PR base. A residual conflict or an unmeasurable check declines before the +large DB artifacts are downloaded. If it passes, Tier 2 uses the original forge PR payload. +`--coverage-max-drift` applies a second fail-closed bound: beyond 30 commits Tier 2 declines and +the PR runs in full. ### 8.3 What happens with the result -`--prepare DIR` does the whole fetch in one call: resolve the PR base, select a complete pair, -stream both tarballs down, unpack their identically named SQLite files separately, and union them -with `compact_db.merge_databases`. It writes the selection JSON beside the merged SQLite as -`cbts_coverage_db.json` and prints `{path, meta}`. Groovy is left with the two things only it can do -— bind the credential and run `coverage_audit.py` over the result — and any failure anywhere is -caught and non-fatal: no prepared DB is returned, Tier 2 never runs, and the PR gets a full run. - -Those two paths reach `main.py` as `--coverage-db` and `--coverage-db-meta`, so a new selection -field needs no Groovy change. `main.py` records all of it and **gates on the drift**: past +After Tier 1 computes the residual, Groovy resolves and persists a build pin before any large +download. `--prepare DIR --build BUILD --paths-json PATH` then resolves the PR base, validates the +pinned pair's residual compatibility, streams both tarballs down, unpacks their identically named +SQLite files separately, and unions them with `compact_db.merge_databases`. +It writes the selection JSON +beside the merged SQLite as `cbts_coverage_db.json` and prints `{path, meta}`. Groovy binds the +credentials, logs the successful compatibility check, and runs `coverage_audit.py` over the result. +On a residual conflict it returns `{path: null, meta}` so the decline remains observable without +downloading the DB. Any failure is non-fatal: Tier 2 never runs and the PR gets a full run. + +The metadata path always reaches `main.py`; the DB path is added only when compatibility is clean. +`main.py` records all of it and **gates on the drift**: past `--coverage-max-drift` (default 30) the tier declines and the PR runs in full, on the grounds that a DB that far from the PR's base no longer describes who touches what in the code under test. A drift that could not be measured — including a meta file that is missing or unreadable — is @@ -231,11 +264,15 @@ All of it lands in the decision and in OpenSearch: |---|---|---| | `coverage_db_build` | `l_coverage_db_build` | 0 when no DB was consulted | | `coverage_db_commit` | `s_coverage_db_commit` | | -| `coverage_db_lag` | `l_coverage_db_lag` | ranking / overall freshness; `null` / `-1` when unmeasurable | +| `coverage_db_lag` | `l_coverage_db_lag` | overall freshness only; `null` / `-1` when unmeasurable | | `coverage_db_base_commit` | `s_coverage_db_base_commit` | the PR's merge base | | `coverage_db_drift` | `l_coverage_db_drift` | **the gated number**; `null` / `-1` when unmeasurable | -| `coverage_db_drift_status` | `s_coverage_db_drift_status` | `ahead` / `identical` for every selected DB | +| `coverage_db_drift_status` | `s_coverage_db_drift_status` | `ahead` / `behind` / `identical` for every selected DB | | `coverage_freshness` | `s_coverage_freshness` | `ok` / `stale` / `unknown`, empty when no DB | +| `coverage_compatibility` | `s_coverage_compatibility` | `clean` / `conflict` / `unknown` / `not_attempted` | +| `coverage_decline_reason` | `s_coverage_decline_reason` | human-readable Tier-2 decline detail | +| `coverage_decline_category` | `s_coverage_decline_category` | aggregation-safe decline category | +| count of `coverage_residual_files` | `l_coverage_residual_files` | Tier-2 opportunity size | so the decline rate is queryable per verdict rather than only readable in `s_reason`. @@ -256,6 +293,10 @@ so the decline rate is queryable per verdict rather than only readable in `s_rea "coverage_db_base_commit": "9f0da65d...", "coverage_db_drift": 7, "coverage_db_drift_status": "ahead", + "coverage_residual_files": ["tensorrt_llm/example.py"], + "coverage_compatibility": "clean", + "coverage_decline_reason": "", + "coverage_decline_category": "", "coverage_no_diff_files": 0, "reasons": [{"source": "coverage", "impacted": 118, "untrusted": 104, ...}] } diff --git a/jenkins/scripts/cbts/coverage_selection/artifact.py b/jenkins/scripts/cbts/coverage_selection/artifact.py index ad2621d294a1..7af39690dad4 100644 --- a/jenkins/scripts/cbts/coverage_selection/artifact.py +++ b/jenkins/scripts/cbts/coverage_selection/artifact.py @@ -14,19 +14,20 @@ """Resolve which post-merge CBTS touch DBs to use. Candidates are recent builds of `` with both x86 and SBSA -coverage tarballs. A candidate must have collected a revision at or before the -PR base; the candidate closest to that base wins, with build number only a -tie-break. Revision ordering comes from the forge compare API — the CI checkout -is depth-1, so git cannot answer it — and needs `GITHUB_API_TOKEN`, the anonymous -quota being per-IP and exhausted by shared CI egress. +coverage tarballs. The newest complete pair wins. A squashed PR commit must +cherry-pick cleanly for Tier 2's residual paths at that DB's revision; +otherwise Tier 2 declines. Revision +metadata comes from the forge compare API and needs `GITHUB_API_TOKEN`, the +anonymous quota being per-IP and exhausted by shared CI egress. The selected architecture DBs are merged locally through the compact coverage schema, producing the one DB consumed by `main.py`. -Two entry points. `--print-selection` prints `{urls, build, commit, lag, -base_commit, drift, drift_status}` and stops. `--prepare DIR` goes on to -download, unpack, and merge the winner, drop that JSON beside it, and print -`{path, meta}` — the two paths `main.py` needs. +Three entry points. `--resolve-build` prints the newest (or explicitly pinned) +build metadata without checking or downloading the PR diff. `--print-selection` +also checks diff compatibility and stops. `--prepare DIR` downloads, unpacks, +and merges the winner, drops that JSON beside it, and prints the two paths +consumed by CBTS. """ from __future__ import annotations @@ -36,6 +37,7 @@ import os import shutil import sqlite3 +import subprocess import sys import tarfile import tempfile @@ -43,7 +45,7 @@ import urllib.request from functools import lru_cache from pathlib import Path -from typing import Optional +from typing import Literal, Optional sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "coverage_utils")) @@ -65,6 +67,8 @@ COVERAGE_BRANCH = "main" # Read by `compare_distance`; the anonymous quota is unusable from shared CI egress IPs. GITHUB_TOKEN_ENV = "GITHUB_API_TOKEN" +# The normal CI checkout's authenticated repository URL, bound by Jenkins. +COVERAGE_GIT_REPO_ENV = "CBTS_COVERAGE_GIT_REPO" _URM = "https://urm.nvidia.com/artifactory" _GITHUB_COMPARE = "https://api.github.com/repos/NVIDIA/TensorRT-LLM/compare" @@ -75,9 +79,13 @@ _TIMEOUT = 15 # Socket timeout for the tarball itself, which runs to hundreds of MB. _DOWNLOAD_TIMEOUT = 300 +# Timeout for local Git operations and the small fetches used by the patch check. +_GIT_TIMEOUT = 120 # Tarball download attempts. _RETRIES = 3 +_PatchApplyStatus = Literal["clean", "conflict", "unknown"] + def _get(url: str, headers: Optional[dict] = None) -> tuple[Optional[int], Optional[bytes]]: req = urllib.request.Request(url, headers=headers or {}) @@ -204,18 +212,177 @@ def drift(db_commit: str, base_commit: str) -> tuple[Optional[int], str]: return None, "unknown" +def _run_git( + args: list[str], + *, + cwd: Path, + input_data: Optional[bytes] = None, + env: Optional[dict[str, str]] = None, + timeout: int = _GIT_TIMEOUT, +) -> Optional[subprocess.CompletedProcess[bytes]]: + try: + return subprocess.run( + ["git", *args], + cwd=cwd, + input=input_data, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + check=False, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + command = args[0] if args else "command" + print(f"[artifact] git {command} timed out after {timeout}s", file=sys.stderr) + return None + + +def _patch_apply_status( + pr_base_commit: str, + pr_head: str, + db_commit: str, + repo_root: Path = Path("."), + upstream_url: Optional[str] = None, + relevant_paths: Optional[list[str]] = None, +) -> _PatchApplyStatus: + """Check the relevant paths of a squashed PR commit against the DB revision.""" + repo_source = repo_root.resolve() + main_repo = upstream_url or os.environ.get(COVERAGE_GIT_REPO_ENV) + if not main_repo: + print( + f"[artifact] {COVERAGE_GIT_REPO_ENV} is required for the patch check", + file=sys.stderr, + ) + return "unknown" + checked_out_head = _run_git(["rev-parse", "HEAD"], cwd=repo_source) + if ( + checked_out_head is None + or checked_out_head.returncode != 0 + or checked_out_head.stdout.decode("ascii", "replace").strip() != pr_head + ): + print(f"[artifact] checked-out revision is not PR head {pr_head[:10]}", file=sys.stderr) + return "unknown" + with tempfile.TemporaryDirectory(prefix="cbts_patch_check_") as temp_dir: + temp = Path(temp_dir) + repo = temp / "repo" + initialized = _run_git(["init", str(repo)], cwd=temp) + if initialized is None or initialized.returncode != 0: + print("[artifact] could not initialize the patch-check repository", file=sys.stderr) + return "unknown" + + # The PR head is guaranteed to be the checked-out revision. The base and coverage + # commits come from the same authenticated internal mirror as normal CI checkouts. + fetched_head = _run_git( + ["fetch", "--no-tags", "--depth=1", str(repo_source), "HEAD"], cwd=repo + ) + if fetched_head is None or fetched_head.returncode != 0: + print( + f"[artifact] could not load PR head {pr_head[:10]} for patch check", + file=sys.stderr, + ) + return "unknown" + + revisions = list(dict.fromkeys((pr_base_commit, db_commit))) + fetched_revisions = _run_git( + ["fetch", "--no-tags", "--depth=1", main_repo, *revisions], cwd=repo + ) + if fetched_revisions is None or fetched_revisions.returncode != 0: + print( + "[artifact] could not load the base/coverage revisions for patch check", + file=sys.stderr, + ) + return "unknown" + + checked_out_db = _run_git(["checkout", "--detach", db_commit], cwd=repo) + if checked_out_db is None or checked_out_db.returncode != 0: + print("[artifact] could not check out the coverage revision", file=sys.stderr) + return "unknown" + + squashed_pr = _run_git( + [ + "-c", + "user.name=CBTS", + "-c", + "user.email=cbts@nvidia.com", + "commit-tree", + f"{pr_head}^{{tree}}", + "-p", + pr_base_commit, + "-m", + "squashed PR for CBTS compatibility check", + ], + cwd=repo, + ) + if squashed_pr is None or squashed_pr.returncode != 0: + print("[artifact] could not create the squashed PR commit", file=sys.stderr) + return "unknown" + + pr_commit = squashed_pr.stdout.decode("ascii", "replace").strip() + applied = _run_git(["cherry-pick", "--no-commit", pr_commit], cwd=repo) + if applied is None: + return "unknown" + if applied.returncode == 0: + return "clean" + if relevant_paths is None: + return "conflict" + unmerged = _run_git( + ["diff", "--name-only", "--diff-filter=U", "-z"], + cwd=repo, + ) + if unmerged is None or unmerged.returncode != 0: + return "unknown" + conflict_paths = { + path.decode("utf-8", "surrogateescape") for path in unmerged.stdout.split(b"\0") if path + } + if not conflict_paths: + return "unknown" + relevant_conflicts = conflict_paths.intersection(relevant_paths) + if relevant_conflicts: + return "conflict" + print( + "[artifact] ignoring conflict(s) outside the Tier-2 residual: " + + ", ".join(sorted(conflict_paths)), + file=sys.stderr, + ) + return "clean" + + +def _selection_accepts_pr_diff( + sel: dict, + pr_base_commit: str, + pr_head: str, + relevant_paths: Optional[list[str]] = None, +) -> bool: + sel["patch_apply_status"] = _patch_apply_status( + pr_base_commit, + pr_head, + sel["commit"], + relevant_paths=relevant_paths, + ) + if sel["patch_apply_status"] == "clean": + sel["coverage_decline_reason"] = "" + sel["coverage_decline_category"] = "" + return True + sel["coverage_decline_reason"] = ( + "coverage tier declined: Tier-2 residual does not apply cleanly to " + f"selected DB commit {sel['commit'][:10]} ({sel['patch_apply_status']})" + ) + sel["coverage_decline_category"] = f"compatibility_{sel['patch_apply_status']}" + print(f"[artifact] {sel['coverage_decline_reason']}", file=sys.stderr) + return False + + def select_tarball( pr_base_commit: str, artifact_base: str = ARTIFACT_BASE, jenkins_base: str = _JENKINS_BASE, max_probe: int = _MAX_PROBE, ) -> Optional[dict]: - """Closest complete architecture pair collected at or before `pr_base_commit`.""" + """Newest complete architecture pair with measurable PR-base topology.""" build = latest_build_number(jenkins_base) if build is None: print("[artifact] could not resolve latest build number", file=sys.stderr) return None - candidates = [] for b in range(build, max(0, build - max_probe), -1): urls = tarball_urls(b, artifact_base) if not all(_exists(url) for url in urls): @@ -227,57 +394,81 @@ def select_tarball( distance, status = drift(commit, pr_base_commit) if distance is None: print( - f"[artifact] build {b}: ordering against the PR base unknown; skipped", + f"[artifact] latest complete build {b}: topology against the PR base unknown", file=sys.stderr, ) - continue - if status not in ("ahead", "identical"): + return None + if status not in ("ahead", "behind", "identical"): print( - f"[artifact] build {b}: relation to the PR base is {status or 'unknown'}; skipped", + f"[artifact] latest complete build {b}: relation to the PR base is " + f"{status or 'unknown'}", file=sys.stderr, ) - continue - candidates.append( - { - "url": urls[0], - "urls": urls, - "build": b, - "commit": commit, - "base_commit": pr_base_commit, - "drift": distance, - "drift_status": status, - } + return None + selected = { + "url": urls[0], + "urls": urls, + "build": b, + "commit": commit, + "base_commit": pr_base_commit, + "drift": distance, + "drift_status": status, + "lag": compare_distance(commit), + } + if selected["lag"] is None: + print(f"[artifact] build {b}: lag behind main unknown", file=sys.stderr) + return selected + print( + f"[artifact] no complete x86/SBSA coverage pair in the last {max_probe} builds", + file=sys.stderr, + ) + return None + + +def select_build( + build: int, + pr_base_commit: str, + expected_commit: Optional[str] = None, + artifact_base: str = ARTIFACT_BASE, +) -> Optional[dict]: + """Resolve and validate one explicitly pinned coverage build.""" + urls = tarball_urls(build, artifact_base) + if not all(_exists(url) for url in urls): + print(f"[artifact] pinned build {build}: coverage pair is incomplete", file=sys.stderr) + return None + commit = build_commit(build, artifact_base) + if not commit: + print(f"[artifact] pinned build {build}: commit unknown", file=sys.stderr) + return None + if expected_commit is not None and commit != expected_commit: + print( + f"[artifact] pinned build {build}: commit changed from {expected_commit} to {commit}", + file=sys.stderr, ) - if not candidates: + return None + distance, status = drift(commit, pr_base_commit) + if distance is None: print( - f"[artifact] no complete x86/SBSA coverage pair at or before the PR base " - f"in the last {max_probe} builds", + f"[artifact] pinned build {build}: topology against the PR base unknown", file=sys.stderr, ) return None - best = min(candidates, key=lambda c: (c["drift"], -c["build"])) - best["lag"] = compare_distance(best["commit"]) - if best["lag"] is None: + if status not in ("ahead", "behind", "identical"): print( - f"[artifact] build {best['build']}: lag behind main unknown", + f"[artifact] pinned build {build}: relation to the PR base is {status or 'unknown'}", file=sys.stderr, ) - return best - - -def measure_drift(sel: dict, pr_head: Optional[str]) -> dict: - """Add PR-base relation metadata to a pinned selection, in place.""" - sel.setdefault("base_commit", None) - sel.setdefault("drift", None) - sel.setdefault("drift_status", "unknown") - if not pr_head or not sel.get("commit"): - return sel - base = merge_base(pr_head) - if not base: - return sel - sel["base_commit"] = base - sel["drift"], sel["drift_status"] = drift(sel["commit"], base) - return sel + return None + return { + "url": urls[0], + "urls": urls, + "build": build, + "commit": commit, + "base_commit": pr_base_commit, + "drift": distance, + "drift_status": status, + "lag": compare_distance(commit), + } def describe(sel: dict) -> str: @@ -291,7 +482,8 @@ def describe(sel: dict) -> str: return ( f"[artifact] selected build {sel.get('build')}, " f"DB commit {(sel.get('commit') or 'unknown')[:10]}; topology from DB: " - f"PR base {base} ({base_distance}), current main tip ({lag})" + f"PR base {base} ({base_distance}), current main tip ({lag}); " + f"PR diff compatibility: {sel.get('patch_apply_status', 'unknown')}" ) @@ -329,26 +521,47 @@ def extract(tarball: Path, dest: Path) -> bool: return True -def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: +def prepare( + dest_dir: str, + pr_head: Optional[str], + relevant_paths: list[str], + build: Optional[int] = None, + expected_commit: Optional[str] = None, +) -> Optional[dict]: """Resolve, download, and merge the DB pair; `{path, meta}` or None on failure. `meta` is the selection JSON on disk, which `main.py --coverage-db-meta` reads. Paths are relative to the caller's cwd, matching the Groovy caller's `cd ${LLM_ROOT}`. """ if not pr_head: - print("[artifact] PR head is required to select an ancestor coverage DB", file=sys.stderr) + print("[artifact] PR head is required to select a coverage DB", file=sys.stderr) return None pr_base_commit = merge_base(pr_head) if not pr_base_commit: print("[artifact] could not resolve the PR base commit", file=sys.stderr) return None - sel = select_tarball(pr_base_commit) + sel = ( + select_build(build, pr_base_commit, expected_commit=expected_commit) + if build is not None + else select_tarball(pr_base_commit) + ) if sel is None: return None + compatible = _selection_accepts_pr_diff( + sel, + pr_base_commit, + pr_head, + relevant_paths, + ) print(describe(sel), file=sys.stderr) dest = Path(dest_dir) dest.mkdir(parents=True, exist_ok=True) + meta = dest / META_NAME + meta.write_text(json.dumps(sel)) + if not compatible: + return {"path": None, "meta": str(meta)} + db = dest / DB_NAME with tempfile.TemporaryDirectory(prefix="cbts_artifacts_", dir=dest) as temp_dir: temp = Path(temp_dir) @@ -371,43 +584,83 @@ def prepare(dest_dir: str, pr_head: Optional[str]) -> Optional[dict]: return None connection.close() - meta = dest / META_NAME - meta.write_text(json.dumps(sel)) return {"path": str(db), "meta": str(meta)} +def _load_paths_json(path: Optional[str]) -> Optional[list[str]]: + """Load the non-empty JSON list of repository-relative Tier-2 paths.""" + if not path: + return None + try: + data = json.loads(Path(path).read_text()) + except (OSError, json.JSONDecodeError) as e: + print(f"[artifact] residual paths unreadable ({path}): {e}", file=sys.stderr) + return None + if not isinstance(data, list) or not data or not all(isinstance(item, str) for item in data): + print("[artifact] residual paths must be a non-empty JSON string list", file=sys.stderr) + return None + return sorted(set(data)) + + def main(argv: Optional[list[str]] = None) -> int: ap = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) + ap.add_argument( + "--resolve-build", + action="store_true", + help="resolve and print build metadata without checking or downloading the PR diff", + ) ap.add_argument( "--print-selection", action="store_true", - help="resolve and print {urls, build, commit, lag, base_commit, drift, drift_status} as JSON", + help="resolve and print {urls, build, commit, lag, base_commit, drift, " + "drift_status, patch_apply_status} as JSON", ) ap.add_argument( "--build", type=int, default=None, help="pin a build number (skip auto-resolve)" ) + ap.add_argument( + "--expected-commit", + default=None, + help="require an explicitly pinned build to retain this coverage commit", + ) ap.add_argument( "--prepare", metavar="DIR", default=None, - help="resolve, download and merge the architecture DBs into DIR, then print " - "{path, meta} as JSON", + help="resolve, validate, download and merge the architecture DBs into DIR, then " + "print {path, meta} as JSON", ) ap.add_argument( "--pr-head", default=None, - help="required PR head revision; its merge base is the inclusive upper bound " - "for eligible coverage revisions", + help="required PR head revision; its merge base measures DB drift and defines " + "the diff checked against the selected coverage revision", + ) + ap.add_argument( + "--paths-json", + default=None, + help="JSON list of Tier-2 residual paths; required for the compatibility check", ) args = ap.parse_args(argv) - if not args.print_selection and not args.prepare: - ap.error("one of --print-selection / --prepare is required") + if sum((args.resolve_build, args.print_selection, args.prepare is not None)) != 1: + ap.error("exactly one of --resolve-build / --print-selection / --prepare is required") + if args.expected_commit is not None and args.build is None: + ap.error("--expected-commit requires --build") if args.prepare: - ready = prepare(args.prepare, args.pr_head) + relevant_paths = _load_paths_json(args.paths_json) + if relevant_paths is None: + return 1 + ready = prepare( + args.prepare, + args.pr_head, + relevant_paths, + build=args.build, + expected_commit=args.expected_commit, + ) if ready is None: return 1 print(json.dumps(ready)) @@ -421,24 +674,24 @@ def main(argv: Optional[list[str]] = None) -> int: return 1 if args.build is not None: - urls = tarball_urls(args.build) - if not all(_exists(url) for url in urls): - return 1 - commit = build_commit(args.build) - best = { - "url": urls[0], - "urls": urls, - "build": args.build, - "commit": commit, - "lag": compare_distance(commit) if commit else None, - } - measure_drift(best, args.pr_head) - if best["drift_status"] not in ("ahead", "identical"): - return 1 + best = select_build( + args.build, + pr_base_commit, + expected_commit=args.expected_commit, + ) else: best = select_tarball(pr_base_commit) if best is None: return 1 + if args.resolve_build: + print(json.dumps(best)) + return 0 + + relevant_paths = _load_paths_json(args.paths_json) + if relevant_paths is None: + return 1 + if not _selection_accepts_pr_diff(best, pr_base_commit, args.pr_head, relevant_paths): + return 1 print(json.dumps(best)) return 0 diff --git a/jenkins/scripts/cbts/coverage_tier.py b/jenkins/scripts/cbts/coverage_tier.py index bd647fd21105..2b6a89fdca9b 100644 --- a/jenkins/scripts/cbts/coverage_tier.py +++ b/jenkins/scripts/cbts/coverage_tier.py @@ -171,6 +171,23 @@ def _build_narrowing( return removed, dropped, must_run +def coverage_preflight( + pr: PRInputs, + pairs: list[tuple[object, RuleResult]], + handled: set[str], +) -> tuple[list[str], str]: + """Return the Tier-2 residual and any reason it cannot be audited.""" + residual = sorted(set(pr.changed_files) - handled) + if any(result.scope is None for _, result in pairs): + return residual, "coverage tier skipped: a rule forced fallback (scope=null)" + if not residual: + return residual, "coverage tier skipped: no residual (all files handled by rules)" + for path in residual: + if not (path.endswith(".py") and path.startswith("tensorrt_llm/")): + return residual, f"coverage tier declined: non-core-Python residual file: {path}" + return residual, "" + + def apply_coverage_tier( pr: PRInputs, pairs: list[tuple[object, RuleResult]], @@ -182,11 +199,9 @@ def apply_coverage_tier( no_data_policy: str = DEFAULT_NO_DATA_POLICY, ) -> tuple[CoverageTierResult | None, str]: """Return (narrowing, note); narrowing is None when the tier keeps the Tier-1 result.""" - if any(r.scope is None for _, r in pairs): - return None, "coverage tier skipped: a rule forced fallback (scope=null)" - residual = sorted(set(pr.changed_files) - handled) - if not residual: - return None, "coverage tier skipped: no residual (all files handled by rules)" + residual, note = coverage_preflight(pr, pairs, handled) + if note: + return None, note selector = CoverageSelector(db, repo_root, no_data_policy=no_data_policy) cov = selector.decide(residual, pr.diffs) diff --git a/jenkins/scripts/cbts/main.py b/jenkins/scripts/cbts/main.py index c925359bdb11..83a7b2c7e08f 100644 --- a/jenkins/scripts/cbts/main.py +++ b/jenkins/scripts/cbts/main.py @@ -55,6 +55,7 @@ NO_DATA_POLICIES, apply_coverage_tier, compute_coverage_stage_counts, + coverage_preflight, open_db, write_coverage_test_db, ) @@ -146,6 +147,13 @@ class SelectionResult: coverage_db_drift_status: str = "" # Freshness verdict on that drift: ok / stale / unknown; empty when no DB was consulted. coverage_freshness: str = "" + # Files left after Tier 1; the patch compatibility check is scoped to these paths. + coverage_residual_files: list[str] = field(default_factory=list) + # clean / conflict / unknown / not_attempted. + coverage_compatibility: str = "not_attempted" + # Stable classification for coverage-tier fallback telemetry. + coverage_decline_reason: str = "" + coverage_decline_category: str = "" # Residual files the forge API returned no patch for; they fall back to file level. coverage_no_diff_files: int = 0 @@ -169,6 +177,10 @@ def to_json(self) -> str: "coverage_db_drift": self.coverage_db_drift, "coverage_db_drift_status": self.coverage_db_drift_status, "coverage_freshness": self.coverage_freshness, + "coverage_residual_files": list(self.coverage_residual_files), + "coverage_compatibility": self.coverage_compatibility, + "coverage_decline_reason": self.coverage_decline_reason, + "coverage_decline_category": self.coverage_decline_category, "coverage_no_diff_files": self.coverage_no_diff_files, } return json.dumps(data, indent=2, ensure_ascii=False) + "\n" @@ -202,6 +214,28 @@ def _coverage_freshness(drift: Optional[int], max_drift: int) -> tuple[str, str] return "ok", "" +def _coverage_decline_category(reason: str) -> str: + """Map a human-readable Tier-2 decline reason to a stable telemetry category.""" + categories = ( + ("a rule forced fallback", "rule_forced_fallback"), + ("no residual", "no_residual"), + ("non-core-Python residual file", "non_core_python"), + ("freshness unknown", "freshness_unknown"), + ("over the", "freshness_stale"), + ("zero-touch residual file", "zero_touch"), + ("no usable diff", "no_usable_diff"), + ("import-executed change", "import_executed"), + ("unparsable source", "unparsable_source"), + ("closure change", "closure_change"), + ("coverage tier errored", "tier_error"), + ) + return ( + next((category for text, category in categories if text in reason), "other") + if reason + else "" + ) + + def _rule_reason(rule, r) -> dict: """One rule's structured reason entry: `{source, blocks, stages}`.""" return { @@ -456,6 +490,13 @@ def main(argv: Optional[list[str]] = None) -> int: rules = build_rules(yaml_index, stages, repo_root) selector = Selector(stages) result = selector.run(pr, rules) + if result.scope is None: + result.coverage_residual_files, result.coverage_decline_reason = coverage_preflight( + pr, selector.pairs, selector.handled + ) + result.coverage_decline_category = _coverage_decline_category( + result.coverage_decline_reason + ) meta = _load_coverage_db_meta(args.coverage_db_meta) result.coverage_db_build = meta.get("build") @@ -464,12 +505,22 @@ def main(argv: Optional[list[str]] = None) -> int: result.coverage_db_drift = meta.get("drift") result.coverage_db_base_commit = meta.get("base_commit") result.coverage_db_drift_status = meta.get("drift_status") or "" + result.coverage_compatibility = meta.get("patch_apply_status") or "not_attempted" + result.coverage_decline_reason = ( + meta.get("coverage_decline_reason") or result.coverage_decline_reason + ) + result.coverage_decline_category = ( + meta.get("coverage_decline_category") or result.coverage_decline_category + ) if args.coverage_db and result.scope is None: tier = None result.coverage_freshness, note = _coverage_freshness( result.coverage_db_drift, args.coverage_max_drift ) + if note: + result.coverage_decline_reason = note + result.coverage_decline_category = _coverage_decline_category(note) if not note: # the gate passed; a note here means it did not try: db = open_db(args.coverage_db) @@ -519,6 +570,8 @@ def main(argv: Optional[list[str]] = None) -> int: cov_reason ] elif note: + result.coverage_decline_reason = note + result.coverage_decline_category = _coverage_decline_category(note) for x in result.reasons: if isinstance(x, dict) and x.get("source") == "fallback": x["coverage_declined"] = note diff --git a/jenkins/scripts/cbts/tools/dryrun.py b/jenkins/scripts/cbts/tools/dryrun.py index 68467e533a91..cc37fc95ed75 100644 --- a/jenkins/scripts/cbts/tools/dryrun.py +++ b/jenkins/scripts/cbts/tools/dryrun.py @@ -203,6 +203,12 @@ def _fmt_summary( lines.append(f"scopes: {result.get('scopes', [])}") lines.append(f"sanity_required: {result.get('sanity_required')}") lines.append(f"perfsanity_required: {result.get('perfsanity_required')}") + lines.append(f"coverage_compatibility: {result.get('coverage_compatibility')}") + lines.append(f"coverage_decline_category: {result.get('coverage_decline_category')!r}") + lines.append(f"coverage_decline_reason: {result.get('coverage_decline_reason')!r}") + residual = result.get("coverage_residual_files", []) + lines.append(f"coverage_residual_files ({len(residual)}):") + lines.extend(f" - {path}" for path in residual) override = result.get("test_db_dir_override") lines.append(f"test_db_dir_override: {override!r}") stages = result.get("affected_stages", []) @@ -373,9 +379,15 @@ def _write_index( range_expr: Optional[str] = None, ) -> None: counts: dict[Optional[str], int] = {} + compatibility_counts: dict[str, int] = {} + decline_counts: dict[str, int] = {} for _, _, _, _, result, _ in rows: scope = result.get("scope") if "_error" not in result else "ERROR" counts[scope] = counts.get(scope, 0) + 1 + compatibility = result.get("coverage_compatibility") or "not_attempted" + compatibility_counts[compatibility] = compatibility_counts.get(compatibility, 0) + 1 + category = result.get("coverage_decline_category") or "none" + decline_counts[category] = decline_counts.get(category, 0) + 1 trigger = "/bot run --post-merge" if post_merge else "/bot run" if range_expr: @@ -411,6 +423,12 @@ def _sort_key(row): lines += ["", "## Scope distribution", ""] for scope, n in sorted(counts.items(), key=lambda kv: -kv[1]): lines.append(f"- {scope or 'None (fallback)'}: {n}") + lines += ["", "## Coverage compatibility distribution", ""] + for compatibility, n in sorted(compatibility_counts.items(), key=lambda kv: -kv[1]): + lines.append(f"- {compatibility}: {n}") + lines += ["", "## Coverage decline distribution", ""] + for category, n in sorted(decline_counts.items(), key=lambda kv: -kv[1]): + lines.append(f"- {category}: {n}") (out_dir / "INDEX.md").write_text("\n".join(lines) + "\n") diff --git a/jenkins/scripts/cbts/tools/report_cbts_decision.py b/jenkins/scripts/cbts/tools/report_cbts_decision.py index 5970d801efa0..f4f093c3bd77 100644 --- a/jenkins/scripts/cbts/tools/report_cbts_decision.py +++ b/jenkins/scripts/cbts/tools/report_cbts_decision.py @@ -230,6 +230,10 @@ def build_document( "s_coverage_db_drift_status": decision.get("coverage_db_drift_status") or "", # Freshness-gate verdict on that drift: ok / stale / unknown; empty when no DB was consulted. "s_coverage_freshness": decision.get("coverage_freshness") or "", + "s_coverage_compatibility": decision.get("coverage_compatibility") or "not_attempted", + "s_coverage_decline_reason": decision.get("coverage_decline_reason") or "", + "s_coverage_decline_category": decision.get("coverage_decline_category") or "", + "l_coverage_residual_files": len(decision.get("coverage_residual_files") or []), # This field is consumed directly by the CBTS OpenSearch dashboard. "d_case_skip_rate": round(case_skip_rate, 4), "b_case_skip_rate_valid": case_skip_rate_valid, diff --git a/tests/unittest/scripts/test_cbts.py b/tests/unittest/scripts/test_cbts.py index 8df099b1ef85..5738cf3cbf4f 100644 --- a/tests/unittest/scripts/test_cbts.py +++ b/tests/unittest/scripts/test_cbts.py @@ -19,9 +19,11 @@ import ast import importlib.util +import io import json import shutil import sqlite3 +import subprocess import sys import tempfile import types @@ -82,46 +84,88 @@ def _load_main() -> ModuleType: cbts_main = _load_main() +def _git(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + text=True, + timeout=120, + ).stdout.strip() + + class CoverageArtifactTest(unittest.TestCase): - def test_selects_closest_complete_ancestor_pair(self) -> None: - commits = {104: "newer", 102: "older-three", 101: "older-one"} - - def exists(url: str) -> bool: - if "/103/" in url: - return url.endswith("cbts_pystart_report_x86_64.tar.gz") - return "/100/" not in url - - relations = { - "newer": (1, "behind"), - "older-three": (3, "ahead"), - "older-one": (1, "ahead"), - } - with ( - mock.patch.object(artifact, "latest_build_number", return_value=104), - mock.patch.object(artifact, "_exists", side_effect=exists), - mock.patch.object( - artifact, "build_commit", side_effect=lambda build, _base: commits[build] - ), - mock.patch.object( - artifact, "drift", side_effect=lambda commit, _base: relations[commit] - ), - mock.patch.object(artifact, "compare_distance", return_value=7) as lag, - ): - selected = artifact.select_tarball( - "pr-base", artifact_base="coverage", jenkins_base="jenkins", max_probe=5 - ) + def test_patch_apply_status_detects_clean_and_conflicting_diffs(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + repo = Path(temp_dir) + _git(repo, "init") + _git(repo, "config", "user.email", "cbts@example.com") + _git(repo, "config", "user.name", "CBTS Test") + source = repo / "source.py" + waives = repo / "waives.txt" + source.write_text("first\nbase\nlast\n") + waives.write_text("base\n") + _git(repo, "add", "source.py", "waives.txt") + _git(repo, "commit", "-m", "base") + base = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "pr") + source.write_text("first\npr\nlast\n") + waives.write_text("pr\n") + _git(repo, "commit", "-am", "pr") + head = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "db-clean", base) + (repo / "other.py").write_text("coverage revision\n") + _git(repo, "add", "other.py") + _git(repo, "commit", "-m", "non-conflicting db") + clean_db = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "db-conflict", base) + source.write_text("first\ndb\nlast\n") + _git(repo, "commit", "-am", "conflicting db") + conflicting_db = _git(repo, "rev-parse", "HEAD") + + _git(repo, "checkout", "-b", "db-irrelevant-conflict", base) + waives.write_text("db\n") + _git(repo, "commit", "-am", "conflicting non-residual file") + irrelevant_conflict_db = _git(repo, "rev-parse", "HEAD") + _git(repo, "checkout", "pr") - self.assertIsNotNone(selected) - assert selected is not None - self.assertEqual(selected["build"], 101) - self.assertEqual(selected["commit"], "older-one") - self.assertEqual(selected["drift"], 1) - self.assertEqual(selected["drift_status"], "ahead") - self.assertEqual( - [url.rsplit("/", 1)[-1] for url in selected["urls"]], - list(artifact.ARCH_TARBALL_NAMES), - ) - lag.assert_called_once_with("older-one") + self.assertEqual( + artifact._patch_apply_status(base, head, clean_db, repo, str(repo)), "clean" + ) + self.assertEqual( + artifact._patch_apply_status(base, head, conflicting_db, repo, str(repo)), + "conflict", + ) + self.assertEqual( + artifact._patch_apply_status( + base, + head, + irrelevant_conflict_db, + repo, + str(repo), + ["source.py"], + ), + "clean", + ) + self.assertEqual( + artifact._patch_apply_status( + base, + head, + irrelevant_conflict_db, + repo, + str(repo), + ["waives.txt"], + ), + "conflict", + ) + with mock.patch.dict(artifact.os.environ, {artifact.COVERAGE_GIT_REPO_ENV: ""}): + self.assertEqual( + artifact._patch_apply_status(base, head, clean_db, repo), "unknown" + ) def test_accepts_artifact_collected_at_pr_base(self) -> None: with ( @@ -138,6 +182,90 @@ def test_accepts_artifact_collected_at_pr_base(self) -> None: self.assertEqual(selected["drift"], 0) self.assertEqual(selected["drift_status"], "identical") + def test_select_build_resolves_explicit_pinned_build(self) -> None: + with ( + mock.patch.object(artifact, "_exists", return_value=True), + mock.patch.object(artifact, "build_commit", return_value="coverage-commit"), + mock.patch.object(artifact, "drift", return_value=(3, "behind")), + mock.patch.object(artifact, "compare_distance", return_value=7), + ): + selected = artifact.select_build(42, "pr-base") + + self.assertIsNotNone(selected) + assert selected is not None + self.assertEqual(selected["build"], 42) + self.assertEqual(selected["commit"], "coverage-commit") + self.assertEqual(selected["base_commit"], "pr-base") + self.assertEqual(selected["drift"], 3) + + def test_select_build_rejects_changed_pinned_commit(self) -> None: + with ( + mock.patch.object(artifact, "_exists", return_value=True), + mock.patch.object(artifact, "build_commit", return_value="replacement-commit"), + mock.patch.object(artifact, "drift") as drift, + ): + selected = artifact.select_build( + 42, + "pr-base", + expected_commit="pinned-commit", + ) + + self.assertIsNone(selected) + drift.assert_not_called() + + def test_resolve_build_prints_metadata_without_residual_paths(self) -> None: + selection = { + "build": 42, + "commit": "coverage-commit", + "base_commit": "pr-base", + } + stdout = io.StringIO() + with ( + mock.patch.object(artifact, "merge_base", return_value="pr-base"), + mock.patch.object(artifact, "select_tarball", return_value=selection), + mock.patch("sys.stdout", stdout), + ): + status = artifact.main(["--resolve-build", "--pr-head", "pr-head"]) + + self.assertEqual(status, 0) + self.assertEqual(json.loads(stdout.getvalue()), selection) + + def test_prepare_uses_explicit_pinned_build(self) -> None: + selection = { + "url": "x86-url", + "urls": ["x86-url", "sbsa-url"], + "build": 42, + "commit": "coverage-commit", + "base_commit": "pr-base", + "drift": 2, + "drift_status": "behind", + "lag": 5, + } + with ( + tempfile.TemporaryDirectory() as temp_dir, + mock.patch.object(artifact, "merge_base", return_value="pr-base"), + mock.patch.object(artifact, "select_build", return_value=selection) as select_build, + mock.patch.object(artifact, "select_tarball") as select_latest, + mock.patch.object(artifact, "_patch_apply_status", return_value="conflict"), + ): + ready = artifact.prepare( + temp_dir, + "pr-head", + ["tensorrt_llm/source.py"], + build=42, + expected_commit="coverage-commit", + ) + + self.assertIsNotNone(ready) + assert ready is not None + self.assertIsNone(ready["path"]) + select_build.assert_called_once_with( + 42, + "pr-base", + expected_commit="coverage-commit", + ) + select_latest.assert_not_called() + def test_prepare_merges_x86_and_sbsa_databases(self) -> None: with tempfile.TemporaryDirectory() as temp_dir: root = Path(temp_dir) @@ -191,14 +319,21 @@ def extract(tarball: Path, destination: Path) -> bool: with ( mock.patch.object(artifact, "merge_base", return_value="pr-base"), mock.patch.object(artifact, "select_tarball", return_value=selection) as select, + mock.patch.object(artifact, "_patch_apply_status", return_value="clean") as apply, mock.patch.object(artifact, "download", side_effect=download), mock.patch.object(artifact, "extract", side_effect=extract), ): - ready = artifact.prepare(str(output_dir), "pr-head") + ready = artifact.prepare(str(output_dir), "pr-head", ["tensorrt_llm/source.py"]) self.assertIsNotNone(ready) assert ready is not None select.assert_called_once_with("pr-base") + apply.assert_called_once_with( + "pr-base", + "pr-head", + "coverage-commit", + relevant_paths=["tensorrt_llm/source.py"], + ) connection = sqlite3.connect(ready["path"]) try: tests = { @@ -215,12 +350,6 @@ def extract(tarball: Path, destination: Path) -> bool: ) self.assertEqual(json.loads(Path(ready["meta"]).read_text()), selection) - def test_freshness_gate_honors_configured_threshold(self) -> None: - self.assertEqual(cbts_main._coverage_freshness(7, 7), ("ok", "")) - freshness, reason = cbts_main._coverage_freshness(8, 7) - self.assertEqual(freshness, "stale") - self.assertTrue(reason) - # Coverage pilot @@ -500,6 +629,10 @@ def test_build_document_filters_unscheduled_stages_and_persists_valid_rate( "H100-PyTorch-1": 1, "H100-4_GPUs-PyTorch-1": 1, } + decision["coverage_compatibility"] = "conflict" + decision["coverage_decline_reason"] = "residual conflict" + decision["coverage_decline_category"] = "compatibility_conflict" + decision["coverage_residual_files"] = ["tensorrt_llm/source.py"] document = report_module.build_document( decision, @@ -516,6 +649,10 @@ def test_build_document_filters_unscheduled_stages_and_persists_valid_rate( assert document["b_case_skip_rate_valid"] is True assert document["b_non_cbts_multi_gpu_required"] is True assert document["b_multi_gpu_label_gate_open"] is False + assert document["s_coverage_compatibility"] == "conflict" + assert document["s_coverage_decline_reason"] == "residual conflict" + assert document["s_coverage_decline_category"] == "compatibility_conflict" + assert document["l_coverage_residual_files"] == 1 assert document["flat_detail"]["hit_stages"] == [ "H100-PyTorch-1", "H100-PyTorch-2",