[TRTLLM-15885][infra] validate CBTS coverage DB compatibility

jenkins/L0_MergeRequest.groovy

@@ -157,6 +157,12 @@ def INFRA_DRY_RUN = "infra_dry_run"
 // Kill switch for CBTS per-test coverage; official post-merge pipeline only, single-GPU stages only in Phase 1.
 @Field
 def ENABLE_CBTS_COVERAGE = true
+@Field
+def CBTS_COVERAGE_PIN_VERSION = 1
+@Field
+def CBTS_COVERAGE_PIN_BASE = "sw-tensorrt-generic/llm-artifacts/LLM/main/cbts/coverage-db-pins/v1"
+@Field
+def URM_ARTIFACTORY_BASE = "https://urm.nvidia.com/artifactory"
 // Version-controlled Tier 2 rollout policy. Keep this in the infra-owned Groovy
 // boundary so changing who receives coverage-based narrowing requires infra review.
 @Field
@@ -961,9 +967,6 @@ def getCbtsResult(pipeline, testFilter, globalVars)
         // pyyaml is needed by main.py's blocks.py to parse test-db YAMLs.
         sh "apt-get update -qq && apt-get install -y -qq python3-yaml"
 
-        // Download the touch DB only for PRs in the coverage-tier pilot.
-        def coverageDb = _cbtsCoverageAudit(pipeline)
-
         // Ask Python which file patterns need diffs, fetch them.
         def patternsOut = sh(
             script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py --list-needed-diffs",
@@ -995,12 +998,32 @@ def getCbtsResult(pipeline, testFilter, globalVars)
         writeFile file: inputPath, text: inputJson
 
         def mainCmd = "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/main.py cbts_input.json"
-        if (coverageDb) {
-            mainCmd += " --coverage-db ${coverageDb.path} --coverage-db-meta ${coverageDb.meta}"
-        }
         def output = sh(script: mainCmd, returnStdout: true)
-
         def result = _cbtsParseSelectionResult(output)
+
+        // Tier 1 owns the definition of handled files. Only prepare a coverage DB when
+        // its residual is eligible for Tier 2, and scope the compatibility check to it.
+        if (result.scope == null && result.coverage_residual_files &&
+                !result.coverage_decline_reason) {
+            def residualPath = "${LLM_ROOT}/cbts_coverage_residual.json"
+            writeFile file: residualPath,
+                      text: groovy.json.JsonOutput.toJson(result.coverage_residual_files)
+            def coverageDb = _cbtsCoverageAudit(pipeline, globalVars, residualPath)
+            if (coverageDb?.meta) {
+                def coverageCmd = mainCmd + " --coverage-db-meta ${coverageDb.meta}"
+                if (coverageDb.path) {
+                    coverageCmd += " --coverage-db ${coverageDb.path}"
+                }
+                output = sh(script: coverageCmd, returnStdout: true)
+                result = _cbtsParseSelectionResult(output)
+            } else if (coverageDb?.compatibility) {
+                result.coverage_compatibility = coverageDb.compatibility
+                result.coverage_decline_reason = coverageDb.decline_reason ?: ""
+                result.coverage_decline_category = "coverage_unavailable"
+                output = groovy.json.JsonOutput.toJson(result)
+            }
+        }
+
         if (result.scope == null) {
             pipeline.echo("CBTS: deferring — Python returned scope=null. " +
                           "Reasons: ${result.reasons.join('; ')}")
@@ -1061,9 +1084,9 @@ def _cbtsMultiGpuLabelGateOpen(pipeline, globalVars)
     }
 }
 
-// Check pilot eligibility, then fetch and audit the touch DB; artifact.py's
-// {path, meta} verbatim, or null on failure.
-def _cbtsCoverageAudit(pipeline)
+// Check pilot eligibility, then fetch and audit the touch DB. A compatibility
+// decline returns metadata without a DB path so the reason remains observable.
+def _cbtsCoverageAudit(pipeline, globalVars, String residualPath)
 {
     try {
         // artifact.py resolves, downloads and merges the x86/SBSA DBs; paths come back
@@ -1073,30 +1096,157 @@ def _cbtsCoverageAudit(pipeline)
         def readyJson = ""
         def prAuthor = ""
         def pilotEligible = false
-        withCredentials([usernamePassword(credentialsId: 'github-cred-trtllm-ci', usernameVariable: 'NOT_USED_YET', passwordVariable: 'GITHUB_API_TOKEN')]) {
+        withCredentials([usernamePassword(
+            credentialsId: 'github-cred-trtllm-ci',
+            usernameVariable: 'NOT_USED_YET',
+            passwordVariable: 'GITHUB_API_TOKEN'),
+        ]) {
             prAuthor = sh(
                 script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_pilot.py",
                 returnStdout: true,
             ).trim()
             pilotEligible = prAuthor && CBTS_COVERAGE_PILOT_USERS.any { it.equalsIgnoreCase(prAuthor) }
             pipeline.echo("CBTS coverage pilot: pr_author=${prAuthor ?: 'unknown'}, eligible=${pilotEligible}")
-            if (pilotEligible) {
-                readyJson = sh(
+        }
+        if (!pilotEligible) {
+            pipeline.echo("CBTS: coverage tier disabled for this PR — running Tier 1 only")
+            return null
+        }
+
+        def prNumber = _cbtsPrNumber(globalVars)
+        if (!prHead || !prNumber) {
+            pipeline.echo("CBTS audit: PR number/head unavailable; cannot make coverage DB selection repeatable")
+            return [
+                compatibility: "unknown",
+                decline_reason: "coverage tier declined: PR identity unavailable for coverage DB pin",
+            ]
+        }
+
+        def pinPath = "${LLM_ROOT}/cbts_db_pin.json"
+        def pinUrl = "${URM_ARTIFACTORY_BASE}/${CBTS_COVERAGE_PIN_BASE}/${prNumber}/${prHead}/cbts_db_pin.json"
+        if (fileExists(pinPath)) {
+            deleteFile pinPath
+        }
+        def pinStatus = sh(
+            script: "curl --noproxy '*' -sS -o ${pinPath} -w '%{http_code}' " +
+                    "--connect-timeout 5 --max-time 30 '${pinUrl}' || true",
+            returnStdout: true,
+        ).trim()
+        def pinnedBuild = null
+        def pinnedCommit = ""
+        def pinExists = pinStatus == "200"
+        if (pinExists) {
+            def pin = new groovy.json.JsonSlurper().parseText(readFile(file: pinPath))
+            def validPin = pin instanceof Map &&
+                pin.version?.toString() == CBTS_COVERAGE_PIN_VERSION.toString() &&
+                pin.pr_number?.toString() == prNumber.toString() &&
+                pin.pr_head?.toString() == prHead &&
+                pin.coverage_db_build?.toString() ==~ /[1-9]\d*/ &&
+                pin.coverage_db_commit?.toString() ==~ /[0-9a-fA-F]{40}/
+            if (!validPin) {
+                pipeline.echo("CBTS audit: coverage DB pin is malformed or does not match this PR head")
+                return [
+                    compatibility: "unknown",
+                    decline_reason: "coverage tier declined: invalid coverage DB pin",
+                ]
+            }
+            pinnedBuild = pin.coverage_db_build.toString().toInteger()
+            pinnedCommit = pin.coverage_db_commit.toString()
+            pipeline.echo("CBTS audit: reusing pinned coverage DB build ${pinnedBuild}")
+        } else if (pinStatus == "404") {
+            pipeline.echo("CBTS audit: no pin for this PR head; selecting the latest coverage DB")
+        } else {
+            pipeline.echo("CBTS audit: coverage DB pin query failed (HTTP ${pinStatus ?: 'unknown'})")
+            return [
+                compatibility: "unknown",
+                decline_reason: "coverage tier declined: coverage DB pin query failed",
+            ]
+        }
+
+        if (!pinExists) {
+            def selectionJson = ""
+            withCredentials([usernamePassword(
+                credentialsId: 'github-cred-trtllm-ci',
+                usernameVariable: 'NOT_USED_YET',
+                passwordVariable: 'GITHUB_API_TOKEN'),
+            ]) {
+                selectionJson = sh(
                     script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " +
-                            "--prepare cbts_cov${prHead ? " --pr-head ${prHead}" : ""} || true",
+                            "--resolve-build --pr-head ${prHead} || true",
                     returnStdout: true,
                 ).trim()
             }
+            if (!selectionJson) {
+                pipeline.echo("CBTS audit: latest coverage DB could not be resolved; pin not written")
+                return [
+                    compatibility: "unknown",
+                    decline_reason: "coverage tier declined: coverage DB could not be resolved",
+                ]
+            }
+            def selected = new groovy.json.JsonSlurper().parseText(selectionJson)
+            if (!(selected instanceof Map) ||
+                    !(selected.build?.toString() ==~ /[1-9]\d*/) ||
+                    !(selected.commit?.toString() ==~ /[0-9a-fA-F]{40}/)) {
+                pipeline.echo("CBTS audit: selected coverage DB metadata is invalid; pin not written")
+                return [
+                    compatibility: "unknown",
+                    decline_reason: "coverage tier declined: selected coverage DB metadata invalid",
+                ]
+            }
+            def pin = [
+                version: CBTS_COVERAGE_PIN_VERSION,
+                pr_number: prNumber.toString(),
+                pr_head: prHead,
+                coverage_db_build: selected.build,
+                coverage_db_commit: selected.commit,
+            ]
+            writeFile file: pinPath, text: groovy.json.JsonOutput.toJson(pin)
+            try {
+                def pinTarget = "${CBTS_COVERAGE_PIN_BASE}/${prNumber}/${prHead}/"
+                trtllm_utils.uploadArtifacts(pinPath, pinTarget)
+                pinnedBuild = selected.build.toString().toInteger()
+                pinnedCommit = selected.commit.toString()
+                pipeline.echo("CBTS audit: pinned coverage DB build ${pinnedBuild} for PR head ${prHead}")
+            } catch (InterruptedException e) {
+                throw e
+            } catch (Exception e) {
+                pipeline.echo("CBTS audit: coverage DB pin upload failed (${e.message})")
+                return [
+                    compatibility: "unknown",
+                    decline_reason: "coverage tier declined: coverage DB pin upload failed",
+                ]
+            }
         }
-        if (!pilotEligible) {
-            pipeline.echo("CBTS: coverage tier disabled for this PR — running Tier 1 only")
-            return null
+
+        withCredentials([
+            usernamePassword(
+                credentialsId: 'github-cred-trtllm-ci',
+                usernameVariable: 'NOT_USED_YET',
+                passwordVariable: 'GITHUB_API_TOKEN'),
+            string(credentialsId: 'default-llm-repo', variable: 'CBTS_COVERAGE_GIT_REPO'),
+        ]) {
+            readyJson = sh(
+                script: "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/coverage_selection/artifact.py " +
+                        "--prepare cbts_cov --paths-json ${residualPath}" +
+                        " --pr-head ${prHead}" +
+                        " --build ${pinnedBuild}" +
+                        " --expected-commit ${pinnedCommit} || true",
+                returnStdout: true,
+            ).trim()
         }
         if (!readyJson) {
             pipeline.echo("CBTS audit: no coverage DB could be prepared — skipping Tier 2")
-            return null
+            return [
+                compatibility: "unknown",
+                decline_reason: "coverage tier declined: coverage DB could not be prepared",
+            ]
         }
         def ready = new groovy.json.JsonSlurper().parseText(readyJson)
+        if (!ready.path) {
+            pipeline.echo("CBTS audit: Tier-2 residual conflicts with the selected coverage DB")
+            return ready
+        }
+        pipeline.echo("CBTS audit: Tier-2 residual applies cleanly to the selected coverage DB")
         sh "cd ${LLM_ROOT} && python3 jenkins/scripts/cbts/tools/coverage_audit.py --db ${ready.path}"
         return ready
     } catch (InterruptedException e) {
@@ -1107,6 +1257,19 @@ def _cbtsCoverageAudit(pipeline)
     }
 }
 
+def _cbtsPrNumber(globalVars)
+{
+    def prUrl = globalVars[GITHUB_PR_API_URL]
+    if (prUrl) {
+        def match = (prUrl =~ /\/pulls?\/(\d+)/)
+        if (match.find()) {
+            return match.group(1)
+        }
+        return ""
+    }
+    return env.gitlabMergeRequestIid ?: ""
+}
+
 // Post one CBTS decision record to OpenSearch (best-effort; never blocks CI).
 // decisionJson null for deferred; reason used only then. Context/creds via env.
 // Multi-GPU enters the pre-merge denominator only when normal CI requires it
@@ -1131,16 +1294,7 @@ def _cbtsReportDecision(pipeline, globalVars, String status, String reason, Stri
         // PR number for s_pr_number, mirroring perf_regression_utils: GitHub PR
         // builds carry it in github_pr_api_url (.../pulls/<n>); GitLab MR builds
         // expose env.gitlabMergeRequestIid. Empty for post-merge/branch builds.
-        def prNumber = ""
-        def prUrl = globalVars[GITHUB_PR_API_URL]
-        if (prUrl) {
-            def m = (prUrl =~ /\/pulls?\/(\d+)/)
-            if (m.find()) {
-                prNumber = m.group(1)
-            }
-        } else {
-            prNumber = env.gitlabMergeRequestIid ?: ""
-        }
+        def prNumber = _cbtsPrNumber(globalVars)
         if (prNumber) {
             args += " --pr-number ${prNumber}"
         }

jenkins/scripts/cbts/README.md

@@ -325,6 +325,10 @@ CBTS defers to the existing filter chain when:
   decorator line), has no usable patch, has unparsable source, or has a closure
   change with no wider row set (see `coverage_selection/SELECTION.md` §3-4)
 - No touch DB artifact could be resolved — Tier 2 never runs
+- The Tier-2 residual cannot be applied without conflicts to the selected coverage DB (the
+  freshest complete build on a PR head's first run, then pinned for that head)
+  revision (or the check cannot be completed) — Tier 1-owned file conflicts are ignored, while
+  residual conflicts decline Tier 2 before download
 - The resolved DB sits more than `--coverage-max-drift` commits from the PR's
   base commit, on either side, or an unmeasurable distance from it — Tier 2
   declines (`coverage_freshness` = `stale` / `unknown`)