From e4b0ddc6385738fb74689b50b4a2e24641134ae5 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Thu, 24 Sep 2026 15:07:23 +0800 Subject: [PATCH] [None][infra] Add CodeRabbit semantic conflict checks Signed-off-by: Yanchao Lu --- .coderabbit.yaml | 39 + .github/coderabbit-semantic-review.md | 121 +++ .../coderabbit_semantic_review.test.js | 800 ++++++++++++++++++ .../coderabbit_semantic_review_request.js | 253 ++++++ .../coderabbit_semantic_review_result.js | 216 +++++ .../coderabbit-semantic-review-tests.yml | 83 ++ .../workflows/coderabbit-semantic-review.yml | 141 +++ AGENTS.md | 5 + 8 files changed, 1658 insertions(+) create mode 100644 .github/coderabbit-semantic-review.md create mode 100644 .github/scripts/coderabbit_semantic_review.test.js create mode 100644 .github/scripts/coderabbit_semantic_review_request.js create mode 100644 .github/scripts/coderabbit_semantic_review_result.js create mode 100644 .github/workflows/coderabbit-semantic-review-tests.yml create mode 100644 .github/workflows/coderabbit-semantic-review.yml diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 2101ac66113d..9228047f0411 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -77,6 +77,45 @@ reviews: drafts: false base_branches: ["main", "release/.+"] + # Scheduled/on-demand evaluation is requested by coderabbit-semantic-review.yml. + # Keep this off during ordinary reviews so they cannot bypass its cost policy. + pre_merge_checks: + custom_checks: + - name: "Semantic conflict with target branch" + mode: "off" + instructions: | + Verify the requested head, target and merge base with Git. Compare both + diffs from the merge base and inspect the combined code. Use only those + revisions and source evidence; do not infer compatibility from earlier + reviews. + + First audit the tests: trace new or modified fixtures, fakes, mocks, + subclasses and monkeypatch replacements to the production functions they + exercise, in both directions across the two diffs. For each affected path, + compare actual call arguments, keywords and required attributes with the + replacement implementation. Report test-only incompatibilities too. Example: + run(x, trace=None) is incompatible with a replacement run(x); accepting + trace or a verified adapter resolves it. + + Then audit other affected contracts: return units, defaults, tensor + shapes/dtypes, resource lifetimes and synchronization. Exclude unrelated + pre-existing defects and style. Do not stop after a different unsupported + feature combination. + + FAIL if combining the branches breaks a concrete contract. Give the + triggering call/input and cite both sides. PASS requires a coverage summary + identifying inspected test and production paths and why they remain + compatible. Return INCONCLUSIVE for missing necessary evidence. Do not run + repository code. + + Reply in a normal PR chat comment, not a custom-check table. Start with + SEMANTIC_REVIEW_V3 on its own line. Include the full evidence for every + verdict, including PASS, using immutable GitHub blob links with full SHAs + and line numbers from head and target. Include this record on one line: + SEMANTIC_RESULT head= target= merge_base= verdict= + Use verified full lowercase SHAs; omit the record if unavailable. A record + alone is insufficient. + path_filters: # Vendored/adapted FlashInfer kernels; excluded from review. - "!tensorrt_llm/_torch/attention/backends/prims_ts/**" diff --git a/.github/coderabbit-semantic-review.md b/.github/coderabbit-semantic-review.md new file mode 100644 index 000000000000..833c9935c461 --- /dev/null +++ b/.github/coderabbit-semantic-review.md @@ -0,0 +1,121 @@ + + +# Advisory semantic conflict review + +The `CodeRabbit Semantic Conflict Review` workflow performs best-effort semantic +compatibility analysis for open, non-draft PRs targeting `main` or `release/**`. +It compares both branches from their merge base and +follows affected callers, contracts, configuration, and tests across files. + +- PR creation, reopening, commit updates, becoming ready, target-branch changes + and six-hour scans evaluate the same threshold. +- The first analysis needs new target commits and either 24 hours since the + merge-base commit or at least 30 target commits beyond that base. After a + completed PASS/FAIL analysis, a changed head or target qualifies after 24 hours + from that reply or 30 additional target commits. A head-only change therefore + qualifies after 24 hours even when the target has not advanced. If the latest + request has no valid verdict, use its request time and target as the baseline. + Rewritten target history falls back to the merge-base threshold. +- An authorized `ci: full pre-merge approved` label or enabling auto-merge + bypasses the threshold. Approval-label authors are checked against the existing + `trt-llm-ci-approvers` team using its existing token. Automatic pre-merge requests + for each PR and target branch share a one-hour cooldown, even across SHA changes. A signal + during cooldown is skipped; ordinary scans and the post-merge audit remain. +- Exact revision pairs are deduplicated. When a reply is missing, incomplete or + Inconclusive, a scheduled scan may retry that version once, at least six hours + after its latest request. This applies to pre-merge analysis and audits. A + verified FAIL is a completed analysis, not a reason to retry. After the one + automatic retry, the same version needs a manual retry; changed versions are + evaluated under the ordinary threshold and get their own retry allowance. + Workflow dispatch accepts one PR number and bypasses the thresholds, cooldown + and deduplication for that PR only. +- Merge events request a post-merge audit regardless of thresholds or cooldown. + The six-hour scan recovers merges from the preceding 24 hours. The audit pins + the actual merge commit and historical target, including for release PRs; + later target updates do not invalidate it. Squash-only rules or the two-parent + merge must establish the historical target; ambiguous rebase history is rejected. + A pre-merge result/request can be reused only when its head/target pair and + GitHub's recorded test-merge tree match the actual merged tree. Otherwise the + audit includes the actual merged code in a new analysis request. + +## Request transport and scan limits + +The workflow reads the semantic instructions from `.coderabbit.yaml`; the native +custom check remains `off` during ordinary reviews. It requests an ordinary +CodeRabbit PR chat reply with `SEMANTIC_REVIEW_V3`, the full revision record and +source links. Native custom-check PASS table cells can truncate those details. +Legacy table replies remain readable but truncated evidence never becomes PASS. +The request explicitly forbids formal review submission or Request Changes. + +Commands use the existing `TRTLLM_AGENT_SHARED_TOKEN` and require the +`trtllm-agent` User account (ID `296075020`), verified before scanning. There is +no fallback to `github-actions[bot]`, whose commands may be ignored. Repository +reads and Check publication still use `GITHUB_TOKEN`; neither token executes PR +code. Acceptance of the service-account commands, especially on merged/release +PRs, requires a deployment pilot. Missing replies never imply semantic success. + +Scheduled scans run at minute 23 every six hours (UTC). Each scan sends at most +20 new AI requests, including audits, to avoid an initial burst across all open +PRs. Recent merged PRs are considered first; open PRs use a rotating starting +point. The single automatic retry also counts toward the 20-request limit. +Scans restart from live state, not a saved cursor. Saturation can delay PRs; +use a manual dispatch for a specific PR rather than relying on a resume guarantee. + +Each token's remaining REST budget is read once, then tracked from response +headers, including pagination. A scan stops below a 100-request reserve or on +primary/secondary rate limiting and reports processed PRs and new requests. +Ordinary permission errors remain failures. The 20-request cap and 100-request +reserve apply only to scheduled scans; GitHub's own rate limits can affect every +API operation, including single-PR events, manual requests and result publication. +Frequency reduction does not reduce the peak cost of one scan. A rate-limited or missed scan can also delay +post-merge recovery beyond its 24-hour lookback. + +## Result verification + +The `Semantic conflict with target branch` Check starts neutral. Stale results +become neutral when the PR event or scheduled scan observes a version change. +This rule applies to every pre-merge verdict; historical audit verdicts stay +attached to their fixed merged revisions. Only formatted CodeRabbit analysis +replies update the Check. Receiving a reply publishes a result, not another AI +request. +The verifier checks the bot identity, most recent trusted request, exact revision +record, and GitHub merge base. Publication and preview select the newest applicable +reply after that request; delayed events cannot restore an older verdict, and +pending manual retries cannot reuse an earlier PASS. Before deployment, the preview +also accepts manual evaluations when no trusted request exists for the pair. +PASS becomes success; FAIL makes the Check and +publishing job red; Inconclusive remains neutral. A successful request job only +means orchestration succeeded. Checks on the actual merge SHA use the distinct +`Semantic conflict audit (post-merge)` name, with a receipt linking the analysis +on the original PR. Evidence includes code locations and regression scenarios. +The instructions first discover cross-branch interactions, then verify their +contracts, including test replacements and the production paths they exercise. +Replies must cite immutable source links +with full SHAs and line numbers from both head and target. A PASS/FAIL without +those citations becomes Inconclusive, including in the preview; an older PASS +cannot substitute for that incomplete reply. Citation presence does not prove +the AI's reasoning or the cited code is correct. + +CodeRabbit can make mistakes, including false positives. Keep these checks and +workflows non-required: their failures then do not block merging. No required +waiting gate is added, and auto-merge does not wait for this analysis. Audit does +not revert code or modify branches. Repository rules remain unchanged. +The thresholds and cooldown limit frequency, not total calls per PR. + +Changes to this automation run the separate read-only `CodeRabbit Semantic +Review Preview` workflow, including fork drafts. `Automation tests and result +lookup (not AI approval)` runs Node tests and reads actual CodeRabbit replies; +`AI verdict (advisory; skipped = unavailable)` runs only for a verified current +PASS/FAIL and is otherwise gray/skipped. `precommit-check.yml` is unchanged. + +Both workflows use the same verifier. Privileged jobs load only trusted default +branch scripts, never PR code. The preview has read-only permissions. Before +merge, a maintainer can use the exported `command(pair)` function in +`.github/scripts/coderabbit_semantic_review_request.js` to prepare a chat request +with fixed `head`, `target`, `mergeBase` and `branch`, then post it on the PR. +After the reply arrives, rerun the **tests and result lookup** job; rerunning only +the AI job reuses old outputs. Preview tests do not establish production trigger, +permission, command-acceptance or post-merge behavior. diff --git a/.github/scripts/coderabbit_semantic_review.test.js b/.github/scripts/coderabbit_semantic_review.test.js new file mode 100644 index 000000000000..d3541c13b3d0 --- /dev/null +++ b/.github/scripts/coderabbit_semantic_review.test.js @@ -0,0 +1,800 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const assert = require('node:assert/strict'); +const {test} = require('node:test'); +const fs = require('node:fs'); +const path = require('node:path'); +const request = require('./coderabbit_semantic_review_request.js'); +const publish = require('./coderabbit_semantic_review_result.js'); +const {AUDIT, requests} = publish; +const COMMAND_USER = {login: 'trtllm-agent', id: 296075020, type: 'User'}; +const APPROVED = 'ci: full pre-merge approved'; +const HEAD = 'a'.repeat(40), BASE = 'b'.repeat(40), NEW_BASE = 'c'.repeat(40); +const MERGED = 'd'.repeat(40), MERGE_BASE = 'e'.repeat(40), TREE = 'f'.repeat(40); +const HOUR = 3600000, DAY = 24 * HOUR; +const NOW = Date.parse('2026-09-17T08:00:00Z'); +const AsyncFunction = Object.getPrototypeOf(async function () {}).constructor; + +function resultBody(verdict, pair) { + const status = {PASS: '✅ Passed', FAIL: '⚠️ Warning', INCONCLUSIVE: '❓ Inconclusive'}[verdict]; + return '\n' + + `| Semantic Conflict With Target Branch | ${status} | Explanation preview |\n` + + '
\nFull details: Semantic Conflict With Target Branch\n' + + `SEMANTIC_RESULT head=${pair.head} target=${pair.target} merge_base=${pair.mergeBase} verdict=${verdict}\n` + + (pair.merged ? `SEMANTIC_MERGED sha=${pair.merged}\n` : '') + + `Caller: https://github.com/example/repo/blob/${pair.head}/caller.py#L12\n` + + `Implementation: https://github.com/example/repo/blob/${pair.target}/callee.py#L25\n` + + 'Code evidence and a minimal regression input.\n
'; +} + +// Execute the production request and publication modules against an in-memory API. +function harness(overrides = {}) { + const pr = {number: 12, state: 'open', merged: false, draft: false, + head: {sha: HEAD}, base: {ref: 'main'}, labels: [], merge_commit_sha: MERGED, + auto_merge: null, ...overrides}; + const prs = [pr], comments = [], checks = [], posted = [], writes = [], calls = []; + const outputs = {}, warnings = [], failures = [], summaries = []; + let target = BASE, now = NOW, count = 30, age = 0, progressCount = 1; + let mergeTree = TREE, expectedMergeBase = MERGE_BASE, afterCompare = () => {}; + let rules = [{type: 'pull_request', parameters: {allowed_merge_methods: ['squash']}}]; + let mergeParents; + const github = { + rest: { + pulls: {list: 'pulls', get: async args => { + calls.push(['pull', args]); + const found = prs.find(p => p.number === args.pull_number); + if (!found) throw Object.assign(new Error('Not Found'), {status: 404}); + return {data: structuredClone(found)}; + }}, + git: { + getRef: async args => { calls.push(['ref', args]); return {data: {object: {sha: target}}}; }, + getCommit: async args => { calls.push(['commit', args]); return {data: { + sha: args.commit_sha, tree: {sha: mergeTree}, parents: mergeParents || + (pr.merged ? [{sha: BASE}] : [{sha: target}, {sha: pr.head.sha}]), + }}; }, + }, + issues: { + listComments: 'comments', + createComment: async args => { + posted.push(args); writes.push(['comment', args]); + const comment = {id: comments.length + 1, issue_number: args.issue_number, body: args.body, + user: {login: 'github-actions[bot]', type: 'Bot'}, created_at: new Date(now).toISOString(), + html_url: `https://github.com/example/repo/pull/${args.issue_number}#${comments.length + 1}`}; + comments.push(comment); return {data: comment}; + }, + }, + checks: { + listForRef: 'checks', create: async args => { + const check = {...args, id: checks.length + 1, app: {slug: 'github-actions'}}; + checks.push(check); writes.push(['create', args]); return {data: check}; + }, update: async args => { + Object.assign(checks.find(c => c.id === args.check_run_id), args); + writes.push(['update', args]); return {data: {}}; + }, + }, + }, + paginate: async (method, args) => { + calls.push([method, args]); + if (method === 'pulls') return prs.filter(p => p.state === args.state); + if (method === 'comments') return comments.filter(c => (c.issue_number || 12) === args.issue_number); + assert.equal(method, 'checks'); + return checks.filter(c => c.head_sha === args.ref && c.name === args.check_name); + }, + request: async (route, args) => { + calls.push([route, args]); + if (route.endsWith('/rules/branches/{branch}')) return {data: rules}; + assert.equal(route, 'GET /repos/{owner}/{repo}/compare/{basehead}'); + const data = args.basehead.endsWith(`...${pr.head.sha}`) ? + {behind_by: count, merge_base_commit: {sha: expectedMergeBase, + commit: {committer: {date: new Date(NOW - age).toISOString()}}}} : + {status: progressCount ? 'ahead' : 'identical', ahead_by: progressCount}; + afterCompare(); return {data}; + }, + }; + github.paginate.iterator = async function* () { + yield {data: prs.filter(p => p.state === 'closed')}; + }; + const createComment = github.rest.issues.createComment; + const commandGithub = {rest: { + users: {getAuthenticated: async () => ({data: {...COMMAND_USER}})}, + issues: {createComment: async args => { + const response = await createComment(args); + response.data.user = {...COMMAND_USER}; + return response; + }}, + }}; + // Exercise the production Octokit hook around API reads, writes and pagination. + for (const client of [github, commandGithub]) { + const hooks = []; + client.hook = { + wrap: (name, fn) => {assert.equal(name, 'request'); hooks.push(fn);}, + remove: (name, fn) => {assert.equal(name, 'request'); hooks.splice(hooks.indexOf(fn), 1);}, + }; + client.remaining = 15000; + client.apiError = null; + client.rest.rateLimit = {get: async () => ({data: {resources: {core: {remaining: client.remaining}}}})}; + const invoke = async fn => { + const perform = async () => { + if (client.apiError) throw client.apiError; + const response = await fn(); + response.headers = {'x-ratelimit-remaining': String(client.remaining)}; + return response; + }; + return hooks.reduceRight((next, hook) => () => hook(next, {}), perform)(); + }; + for (const group of Object.values(client.rest)) { + for (const [name, fn] of Object.entries(group)) { + if (typeof fn === 'function') group[name] = (...args) => invoke(() => fn(...args)); + } + } + if (client.request) { + const original = client.request; + client.request = (...args) => invoke(() => original(...args)); + } + if (client.paginate) { + const original = client.paginate, iterator = original.iterator; + client.paginate = async (...args) => (await invoke(async () => ({data: await original(...args)}))).data; + client.paginate.iterator = async function* (...args) { + for await (const response of iterator(...args)) yield await invoke(async () => response); + }; + } + } + const core = {setOutput: (k, v) => {outputs[k] = v;}, info() {}, + warning: v => warnings.push(v), error: v => warnings.push(v), setFailed: v => failures.push(v), + summary: {addRaw(v) {summaries.push(v); return this;}, async write() {}}}; + const context = {repo: {owner: 'example', repo: 'repo'}, eventName: 'pull_request_target', + payload: {action: 'opened', pull_request: {number: 12, head: {sha: HEAD}}}}; + return {pr, prs, comments, checks, posted, writes, calls, outputs, warnings, failures, summaries, github, commandGithub, core, + setTarget: v => {target = v;}, setNow: v => {now = v;}, setLag: (n, a) => {count = n; age = a;}, + setProgress: v => {progressCount = v;}, setTree: v => {mergeTree = v;}, + setRules: v => {rules = v;}, setParents: v => {mergeParents = v;}, + setMergeBase: v => {expectedMergeBase = v;}, afterCompare: fn => {afterCompare = fn;}, + async run(eventName = 'pull_request_target', action = 'opened', manual = '12', authorized = false) { + process.env.DISPATCH_PULL_NUMBER = manual; + process.env.SEMANTIC_APPROVAL_VALIDATED = String(authorized); + try { + await request({github, commandGithub, core, now, context: {...context, eventName, + payload: {...context.payload, action, label: {name: APPROVED}}}}); + } finally { + delete process.env.DISPATCH_PULL_NUMBER; + delete process.env.SEMANTIC_APPROVAL_VALIDATED; + } + }, + reply(verdict = 'PASS', pair = requests(comments)[0] || + {head: pr.head.sha, target, mergeBase: MERGE_BASE}) { + const comment = {id: comments.length + 1, body: resultBody(verdict, pair), + created_at: new Date(now).toISOString(), user: {login: 'coderabbitai[bot]', type: 'Bot'}, + html_url: `https://github.com/example/repo/pull/12#${comments.length + 1}`}; + comments.push(comment); return comment; + }, + publish: (comment, preview = false) => publish({github, core, context: { + ...context, eventName: preview ? 'pull_request' : 'issue_comment', + payload: {...context.payload, issue: {number: 12}, comment: {id: comment?.id}}, + }}), + }; +} + +for (const [count, age, expected] of [[0, 3 * DAY, 0], [29, DAY - 1, 0], + [30, 0, 1], [1, DAY, 1], [29, DAY, 1]]) { + test(`first analysis: ${count} target commits, age ${age} => ${expected} requests`, async () => { + for (const event of ['pull_request_target', 'schedule']) { + const h = harness(); h.setLag(count, age); await h.run(event); + assert.equal(h.posted.length, expected); + assert.equal(h.checks[0].conclusion, 'neutral'); + if (expected) { + assert.match(h.posted[0].body, /^@coderabbitai\n/); + assert.match(h.posted[0].body, /normal PR chat comment/); + assert.match(h.posted[0].body, /SEMANTIC_REVIEW_V3/); + assert.match(h.posted[0].body, /No AI verdict is asserted/); + assert.ok(h.posted[0].body.includes(HEAD) && h.posted[0].body.includes(BASE)); + } + } + }); +} + +test('creation and ready events use the threshold; drafts and unrelated targets are excluded', async () => { + for (const action of ['opened', 'ready_for_review']) { + const h = harness(); h.setLag(1, 0); await h.run('pull_request_target', action); + assert.equal(h.posted.length, 0); + } + for (const overrides of [{draft: true}, {state: 'closed'}, {base: {ref: 'feature/a'}}]) { + const h = harness(overrides); await h.run(); + assert.equal(h.writes.length, 0); + await assert.rejects(h.run('workflow_dispatch'), /requires a non-draft open or merged/); + } +}); + +test('release PRs use their actual target branch', async () => { + const h = harness({base: {ref: 'release/1.2'}}); await h.run(); + assert.equal(h.posted.length, 1); + assert.ok(h.calls.filter(([kind]) => kind === 'ref').every(([, args]) => args.ref === 'heads/release/1.2')); + const reply = h.reply(); await h.publish(reply); + assert.equal(h.checks[0].conclusion, 'success'); +}); + +test('the request job accepts base-branch edits and skips title, description and absent changes', () => { + const text = fs.readFileSync(path.join(__dirname, '../workflows/coderabbit-semantic-review.yml'), 'utf8'); + const condition = text.match(/request-review:[\s\S]*? if: >-\n((?: {6}[^\n]+\n)+)/)[1]; + // Missing GitHub context properties evaluate as empty; optional chaining models that here. + const evaluate = new Function('github', 'contains', 'fromJSON', + `return Boolean(${condition.replace('github.event.changes.base.ref', 'github.event.changes?.base?.ref')});`); + const github = {repository: 'NVIDIA/TensorRT-LLM', event_name: 'pull_request_target', event: {}}; + const allowed = event => { + github.event = event; + return evaluate(github, (list, value) => list.includes(value), JSON.parse); + }; + for (const changes of [undefined, {}, {title: {from: 'Old title'}}, {body: {from: 'Old body'}}, + {title: {from: 'Old title'}, body: {from: 'Old body'}}, {base: {sha: {from: BASE}}}]) { + assert.equal(allowed({action: 'edited', changes}), false); + } + for (const from of ['main', 'release/1.2']) { + assert.equal(allowed({action: 'edited', changes: {base: {ref: {from}}}}), true); + } + for (const action of ['opened', 'reopened', 'synchronize', 'ready_for_review', 'auto_merge_enabled']) { + assert.equal(allowed({action}), true); + } + for (const event_name of ['schedule', 'workflow_dispatch']) { + github.event_name = event_name; assert.equal(allowed({}), true); + } +}); + +test('a six-hour scan visits eligible PRs; a PR event or manual retry visits only its PR', async () => { + const h = harness(); h.prs.push({...h.pr, number: 13}); await h.run(); + assert.deepEqual(h.posted.map(c => c.issue_number), [12]); + await h.run('schedule'); assert.deepEqual(h.posted.map(c => c.issue_number), [12, 13]); + await h.run('workflow_dispatch', '', '13'); + assert.deepEqual(h.posted.map(c => c.issue_number), [12, 13, 13]); +}); + +test('dispatch rejects malformed or missing PRs without touching other PRs', async () => { + for (const raw of ['', '0', '-1', '12x', '9007199254740992']) { + const h = harness(); await assert.rejects(h.run('workflow_dispatch', '', raw), /positive integer/); + assert.equal(h.writes.length, 0); + } + const h = harness(); await assert.rejects(h.run('workflow_dispatch', '', '99'), /Not Found/); + assert.equal(h.writes.length, 0); + await assert.rejects(h.run('push'), /Unsupported event/); +}); + +test('events and scans count target progress from the last analysis, not the old merge base', async () => { + for (const event of ['pull_request_target', 'schedule']) { + const h = harness(); h.setLag(80, 3 * DAY); await h.run(); h.reply(); + h.setTarget(NEW_BASE); h.setNow(NOW + 2 * HOUR); h.setProgress(29); await h.run(event); + assert.equal(h.posted.length, 1); + assert.equal(h.checks[0].conclusion, 'neutral'); + assert.match(h.checks[0].output.title, /stale/); + h.setProgress(30); await h.run(event); assert.equal(h.posted.length, 2); + } +}); + +test('a PR head update invalidates the verdict without bypassing the target threshold', async () => { + const h = harness(); await h.run(); await h.publish(h.reply()); + h.pr.head.sha = MERGED; h.setNow(NOW + 2 * HOUR); h.setProgress(0); await h.run(); + assert.equal(h.posted.length, 1); + assert.equal(h.checks.at(-1).head_sha, MERGED); + assert.equal(h.checks.at(-1).conclusion, 'neutral'); +}); + +for (const changed of ['head', 'target', 'both']) { + test(`events and scans require 24 hours for a ${changed} change below 30 target commits`, async () => { + for (const branch of ['main', 'release/1.2']) { + for (const verdict of ['PASS', 'FAIL']) { + const h = harness({base: {ref: branch}}); await h.run(); await h.publish(h.reply(verdict)); + h.setNow(NOW + 6 * HOUR); h.setLag(1, 0); + if (changed !== 'target') h.pr.head.sha = MERGED; + if (changed !== 'head') h.setTarget(NEW_BASE); + h.setProgress(changed === 'head' ? 0 : 1); + await h.run('pull_request_target', 'synchronize'); + assert.equal(h.posted.length, 1); + await h.run('schedule'); assert.equal(h.posted.length, 1); + h.setNow(NOW + DAY - 1); await h.run('schedule'); assert.equal(h.posted.length, 1); + h.setNow(NOW + DAY); await h.run(); + assert.equal(h.posted.length, 2); + const pair = requests(h.comments)[0]; + assert.equal(pair.head, changed === 'target' ? HEAD : MERGED); + assert.equal(pair.target, changed === 'head' ? BASE : NEW_BASE); + assert.equal(pair.branch, branch); + assert.equal(pair.reason, '24-hour / 30-commit threshold'); + assert.equal(h.checks.at(-1).conclusion, 'neutral'); + h.setNow(NOW + DAY + HOUR); await h.run('schedule'); + assert.equal(h.posted.length, 2); // Await the new pair's reply. + } + } + }); +} + +test('scheduled refresh deduplicates an unchanged completed pair across scans', async () => { + for (const verdict of ['PASS', 'FAIL']) { + const h = harness(); await h.run(); await h.publish(h.reply(verdict)); + for (const elapsed of [6 * HOUR, DAY, 2 * DAY]) { + h.setNow(NOW + elapsed); await h.run('schedule'); + assert.equal(h.posted.length, 1); + assert.equal(h.checks[0].conclusion, verdict === 'PASS' ? 'success' : 'failure'); + } + } +}); + +test('scheduled refresh respects cooldown even when 30 target commits qualify', async () => { + const h = harness(); await h.run(); h.reply(); + h.setTarget(NEW_BASE); h.setProgress(30); h.setNow(NOW + HOUR - 1); + await h.run('schedule'); assert.equal(h.posted.length, 1); + assert.match(h.checks.at(-1).output.summary, /cooldown/); + h.setNow(NOW + HOUR); await h.run('schedule'); + assert.equal(h.posted.length, 2); +}); + +test('new revisions use the latest request time when that request has no valid reply', async () => { + const h = harness(); await h.run(); h.reply(); h.setNow(NOW + 2 * HOUR); + await h.run('workflow_dispatch'); + h.pr.head.sha = MERGED; h.setProgress(0); h.setNow(NOW + DAY); + await h.run('schedule'); assert.equal(h.posted.length, 2); + h.setNow(NOW + DAY + 2 * HOUR); await h.run('schedule'); + assert.equal(h.posted.length, 3); +}); + +test('approval and auto-merge bypass thresholds but share a one-hour cooldown across SHAs', async () => { + const h = harness({labels: [{name: APPROVED}], auto_merge: {enabled_by: 'maintainer'}}); + h.setLag(0, 0); await h.run('pull_request_target', 'labeled', '12', true); + assert.equal(h.posted.length, 1); + h.setTarget(NEW_BASE); h.setNow(NOW + HOUR - 1); + await h.run('pull_request_target', 'auto_merge_enabled'); + assert.equal(h.posted.length, 1); assert.equal(h.checks.at(-1).conclusion, 'neutral'); + h.setNow(NOW + HOUR); await h.run('pull_request_target', 'auto_merge_enabled'); + assert.equal(h.posted.length, 2); +}); + +test('an unvalidated approval label cannot bypass the threshold', async () => { + const h = harness({labels: [{name: APPROVED}]}); h.setLag(0, 0); + await h.run('pull_request_target', 'labeled'); assert.equal(h.posted.length, 0); +}); + +test('pending exact pairs deduplicate, manual retry bypasses cooldown and clears a pass', async () => { + const h = harness(); await h.run(); await h.run(); assert.equal(h.posted.length, 1); + await h.publish(h.reply()); assert.equal(h.checks[0].conclusion, 'success'); + await h.run('workflow_dispatch'); assert.equal(h.posted.length, 2); + assert.equal(h.checks[0].conclusion, 'neutral'); +}); + +test('missing analysis cannot repeatedly spend AI calls as the target changes each hour', async () => { + const h = harness(); await h.run(); h.setNow(NOW + 2 * HOUR); h.setTarget(NEW_BASE); + await h.run('schedule'); assert.equal(h.posted.length, 1); +}); + +test('untrusted or malformed request markers cannot suppress analysis', async () => { + const h = harness(); await h.run(); h.comments[0].user.login = 'someone-else'; + h.comments.push({id: 100, user: {login: 'github-actions[bot]', type: 'Bot'}, + body: ''}); + await h.run(); assert.equal(h.posted.length, 2); +}); + +for (const verdict of ['PASS', 'FAIL', 'INCONCLUSIVE']) { + test(`verified ${verdict} maps to the advisory conclusion and notices`, async () => { + const h = harness(); await h.run(); const reply = h.reply(verdict); + reply.body = reply.body.toLowerCase(); await h.publish(reply); + assert.equal(h.checks[0].conclusion, {PASS: 'success', FAIL: 'failure', INCONCLUSIVE: 'neutral'}[verdict]); + assert.equal(h.failures.length, verdict === 'FAIL' ? 1 : 0); + assert.match(h.checks[0].output.summary, /false positives/); + assert.match(h.checks[0].output.summary, /non-required; its failure does not block merging/); + }); +} + +test('malformed, contradictory, stale and untrusted results never publish a pass', async () => { + for (const corrupt of [ + c => {c.user.login = 'attacker';}, c => {c.user.type = 'User';}, + c => {c.body = c.body.replace('', '');}, + c => {c.body = c.body.replace('✅ Passed', '❓ Inconclusive');}, + c => {c.body = c.body.replaceAll(HEAD, NEW_BASE);}, + c => {c.body = c.body.replaceAll(MERGE_BASE, NEW_BASE);}, + c => {c.body = c.body.replace('', c.body.replace('PASS', 'FAIL') + '');}, + ]) { + const h = harness(); await h.run(); const reply = h.reply(); corrupt(reply); await h.publish(reply); + assert.equal(h.checks[0].conclusion, 'neutral'); + } + const h = harness(); await h.run(); const reply = h.reply(); h.comments.shift(); + await h.publish(reply); assert.equal(h.checks[0].conclusion, 'neutral'); +}); + +test('unsupported PASS/FAIL becomes inconclusive instead of reusing an older verdict', async () => { + for (const verdict of ['PASS', 'FAIL']) { + for (const removeEvidence of [ + body => body.replace(/https:\/\/github\.com\/example\/repo\/blob\/\S+/g, ''), + body => body.replace(`/blob/${BASE}/`, `/blob/${NEW_BASE}/`), + body => body.replaceAll('/example/repo/blob/', '/unrelated/repo/blob/'), + body => body.replace(/#L\d+/g, ''), + ]) { + const h = harness(); await h.run(); await h.publish(h.reply('PASS')); + const reply = h.reply(verdict); reply.body = removeEvidence(reply.body); + await h.publish(reply); + assert.equal(h.checks[0].conclusion, 'neutral'); + assert.match(h.checks[0].output.summary, /source links.*both revisions/); + assert.equal(h.failures.length, 0); + await h.publish(null, true); + assert.equal(h.outputs.verdict, 'INCONCLUSIVE'); + } + } +}); + +test('a result after its explanation is accepted with immutable citations', async () => { + const h = harness(); await h.run(); const reply = h.reply('FAIL'); + const record = reply.body.match(/SEMANTIC_RESULT[^\n]+\n/)[0]; + reply.body = reply.body.replace(record, '').replace('', `${record}`); + await h.publish(reply); + assert.equal(h.checks[0].conclusion, 'failure'); +}); + +test('live ref changes during verification cannot publish a current pass', async () => { + for (const change of [h => {h.pr.head.sha = NEW_BASE;}, h => h.setTarget(NEW_BASE)]) { + const h = harness(); await h.run(); const reply = h.reply(); + h.afterCompare(() => change(h)); await h.publish(reply); + assert.equal(h.checks[0].conclusion, 'neutral'); + } +}); + +test('post-merge audit ignores thresholds and cooldown and pins the historical target', async () => { + const h = harness({merged: true, state: 'closed', base: {ref: 'release/1.2'}}); + h.setTarget(NEW_BASE); h.setLag(0, 0); await h.run('pull_request_target', 'closed'); + const pair = requests(h.comments)[0]; + assert.equal(pair.target, BASE); assert.equal(pair.merged, MERGED); + assert.equal(h.checks[0].name, AUDIT); assert.equal(h.checks[0].head_sha, MERGED); + assert.ok(!h.calls.some(([kind]) => kind === 'ref')); + const reply = h.reply('FAIL'); await h.publish(reply); + assert.equal(h.checks[0].conclusion, 'failure'); + assert.match(h.posted.at(-1).body, /Post-merge semantic audit: FAIL/); + await h.publish(reply); assert.equal(h.posted.length, 2); // No duplicate audit receipt. +}); + +test('a matching pre-merge result and actual tree can serve the audit without another AI call', async () => { + const h = harness(); await h.run(); const reply = h.reply(); await h.publish(reply); + h.pr.merged = true; h.pr.state = 'closed'; h.setTarget(NEW_BASE); + await h.run('pull_request_target', 'closed'); + assert.equal(h.posted.filter(c => c.body.startsWith('@coderabbitai')).length, 1); + assert.equal(h.checks.at(-1).name, AUDIT); assert.equal(h.checks.at(-1).conclusion, 'success'); +}); + +test('a matching pre-merge request in flight can complete the audit after merge', async () => { + const h = harness(); await h.run(); const original = requests(h.comments)[0]; + h.pr.merged = true; h.pr.state = 'closed'; await h.run('pull_request_target', 'closed'); + assert.equal(h.posted.length, 1); assert.equal(h.checks.at(-1).conclusion, 'neutral'); + await h.publish(h.reply('PASS', original)); assert.equal(h.checks.at(-1).conclusion, 'success'); +}); + +test('a changed final tree, changed target or inconclusive pre-merge result requires fresh audit', async () => { + for (const scenario of ['tree', 'target', 'inconclusive']) { + const h = harness(); await h.run(); h.reply(scenario === 'inconclusive' ? 'INCONCLUSIVE' : 'PASS'); + h.pr.merged = true; h.pr.state = 'closed'; + if (scenario === 'tree') h.setTree(NEW_BASE); + if (scenario === 'target') h.setParents([{sha: NEW_BASE}]); + await h.run('pull_request_target', 'closed'); + assert.equal(h.posted.length, 2); + assert.equal(requests(h.comments)[0].merged, MERGED); + } +}); + +test('audit refuses an unverified merge method and an audit reply for another merged commit', async () => { + const h = harness({merged: true, state: 'closed'}); h.setRules([]); + await assert.rejects(h.run('pull_request_target', 'closed'), /squash-only/); + assert.equal(h.posted.length, 0); + h.setParents([{sha: BASE}, {sha: HEAD}]); await h.run('pull_request_target', 'closed'); + const reply = h.reply(); reply.body = reply.body.replace(`sha=${MERGED}`, `sha=${NEW_BASE}`); + await h.publish(reply); assert.equal(h.checks[0].conclusion, 'neutral'); +}); + +test('read-only draft preview accepts only a verified current pair and never writes', async () => { + for (const verdict of ['PASS', 'FAIL', 'INCONCLUSIVE']) { + const h = harness({draft: true}); const reply = h.reply(verdict); await h.publish(reply, true); + assert.equal(h.outputs.verdict, verdict); assert.equal(h.writes.length, 0); + assert.equal(h.failures.length, 0); assert.match(h.outputs.summary, /Verified head/); + } + const h = harness({draft: true}); h.reply('PASS'); h.setTarget(NEW_BASE); await h.publish(null, true); + assert.equal(h.outputs.verdict, 'INCONCLUSIVE'); assert.equal(h.writes.length, 0); +}); + +test('the real preview display script fails for conflicts and keeps the advisory notice', async () => { + const text = fs.readFileSync(path.join(__dirname, '../workflows/coderabbit-semantic-review-tests.yml'), 'utf8'); + const script = [...text.matchAll(/^ {10}script: \|\n((?: {12}[^\n]*(?:\n|$)|\n)*)/gm)].at(-1)[1] + .replace(/^ {12}/gm, ''); + const display = new AsyncFunction('core', 'process', script); + for (const verdict of ['PASS', 'FAIL']) { + const h = harness({draft: true}); await h.publish(h.reply(verdict), true); + await display(h.core, {env: {SEMANTIC_VERDICT: h.outputs.verdict, + SEMANTIC_MESSAGE: h.outputs.message, SEMANTIC_SUMMARY: h.outputs.summary}}); + assert.equal(h.failures.length, verdict === 'FAIL' ? 1 : 0); + assert.match(h.summaries.at(-1), /false positives/); + } +}); + +test('a manual retry cannot be satisfied by an older comment event', async () => { + const h = harness(); await h.run(); const oldReply = h.reply(); await h.publish(oldReply); + h.setNow(NOW + HOUR); await h.run('workflow_dispatch'); await h.publish(oldReply); + assert.equal(h.checks[0].conclusion, 'neutral'); + await h.publish(h.reply('FAIL')); assert.equal(h.checks[0].conclusion, 'failure'); +}); + +test('delayed result events reconcile the newest verdict for open PRs and audits', async () => { + for (const merged of [false, true]) { + for (const verdict of ['PASS', 'FAIL', 'INCONCLUSIVE']) { + const h = harness({merged, state: merged ? 'closed' : 'open'}); + await h.run(); + const older = h.reply(verdict === 'PASS' ? 'FAIL' : 'PASS'); + const newer = h.reply(verdict); + await h.publish(newer); + await h.publish(older); + assert.equal(h.checks[0].conclusion, + {PASS: 'success', FAIL: 'failure', INCONCLUSIVE: 'neutral'}[verdict]); + assert.equal(h.checks[0].details_url, newer.html_url); + if (merged) assert.equal(h.posted.length, 2); // Request and one current audit receipt. + } + } +}); + +test('preview waits for the latest manual retry before accepting a verdict', async () => { + const h = harness(); await h.run(); await h.publish(h.reply('PASS')); + h.setNow(NOW + HOUR); await h.run('workflow_dispatch'); + const writes = h.writes.length; + await h.publish(null, true); + assert.equal(h.outputs.verdict, 'INCONCLUSIVE'); + assert.equal(h.writes.length, writes); + await h.publish(h.reply('FAIL'), true); + assert.equal(h.outputs.verdict, 'FAIL'); + assert.equal(h.writes.length, writes); +}); + +test('a new audit request cannot be overwritten by an older pre-merge result', async () => { + const h = harness(); await h.run(); const oldReply = h.reply('INCONCLUSIVE'); + h.pr.merged = true; h.pr.state = 'closed'; await h.run('pull_request_target', 'closed'); + await h.publish(h.reply('FAIL')); oldReply.body = oldReply.body.replaceAll('INCONCLUSIVE', 'PASS') + .replace('❓ Inconclusive', '✅ Passed'); + await h.publish(oldReply); assert.equal(h.checks.at(-1).conclusion, 'failure'); +}); + +test('the six-hour scan recovers recent merged PRs without auditing old closed history', async () => { + const h = harness({merged: true, state: 'closed', merged_at: new Date(NOW - HOUR).toISOString(), + updated_at: new Date(NOW).toISOString()}); + h.prs.push({...h.pr, number: 13, merged_at: new Date(NOW - 2 * DAY).toISOString()}); + await h.run('schedule'); assert.deepEqual(h.posted.map(c => c.issue_number), [12]); +}); + +test('a manual retry reports when refs change before the request is posted', async () => { + const h = harness(); h.afterCompare(() => h.setTarget(NEW_BASE)); + await assert.rejects(h.run('workflow_dispatch'), /Manual retry not posted/); + assert.equal(h.posted.length, 0); +}); + +test('crossing the time threshold updates a previously waiting Check to awaiting analysis', async () => { + const h = harness(); h.setLag(1, 0); await h.run(); + h.setNow(NOW + DAY); await h.run(); + assert.equal(h.checks.length, 1); assert.equal(h.posted.length, 1); + assert.equal(h.checks[0].output.title, 'Awaiting CodeRabbit analysis'); +}); + +test('approval validation checks the current label, actor and active membership', async () => { + const workflow = fs.readFileSync(path.join(__dirname, '../workflows/coderabbit-semantic-review.yml'), 'utf8'); + const script = [...workflow.matchAll(/^ {10}script: \|\n((?: {12}[^\n]*(?:\n|$)|\n)*)/gm)][0][1] + .replace(/^ {12}/gm, ''); + const validate = new AsyncFunction('github', 'context', 'core', script); + for (const [label, actor, membership, expected] of [ + [true, 'maintainer', 'active', true], [false, 'maintainer', 'active', false], + [true, 'different-user', 'active', false], [true, 'maintainer', 'pending', false], + [true, 'maintainer', 404, false], + ]) { + const github = {rest: { + pulls: {get: async () => ({data: {number: 12, labels: label ? [{name: APPROVED}] : []}})}, + issues: {listEventsForTimeline: 'timeline'}, + teams: {getMembershipForUser: async args => { + assert.equal(args.username, 'maintainer'); assert.equal(args.team_slug, 'trt-llm-ci-approvers'); + if (membership === 404) throw Object.assign(new Error('Not Found'), {status: 404}); + return {data: {state: membership}}; + }}, + }, paginate: async () => [{event: 'labeled', label: {name: APPROVED}, actor: {login: actor}}]}; + const actual = await validate(github, {repo: {owner: 'example', repo: 'repo'}, + payload: {pull_request: {number: 12}, sender: {login: 'maintainer'}}}, {warning() {}}); + assert.equal(actual, expected); + } +}); + +function chatReply(reply) { + const details = reply.body.match(/Full details:[^\n]+\n([\s\S]*?)<\/details>/)[1]; + reply.body = `SEMANTIC_REVIEW_V3\n${details}`; + return reply; +} + +test('full chat PASS and FAIL retain source evidence beyond the native 201-character cell', async () => { + for (const verdict of ['PASS', 'FAIL']) { + const h = harness(); await h.run(); + const reply = chatReply(h.reply(verdict)); + assert.ok(reply.body.length > 201); + await h.publish(reply); + assert.equal(h.checks[0].conclusion, verdict === 'PASS' ? 'success' : 'failure'); + await h.publish(null, true); assert.equal(h.outputs.verdict, verdict); + } +}); + +test('chat transport preserves author, revision, citation and unique-record validation', async () => { + for (const corrupt of [ + c => {c.user.type = 'User';}, c => {c.user.login = 'someone-else';}, + c => {c.body = c.body.replace('SEMANTIC_REVIEW_V3', '');}, + c => {c.body = c.body.replaceAll(HEAD, NEW_BASE);}, + c => {c.body = c.body.replaceAll(MERGE_BASE, NEW_BASE);}, + c => {c.body = c.body.replace(/https:\/\/github.com\/\S+/g, '');}, + c => {c.body += c.body.replace('verdict=PASS', 'verdict=FAIL');}, + ]) { + const h = harness(); await h.run(); + const reply = chatReply(h.reply()); corrupt(reply); await h.publish(reply); + assert.equal(h.checks[0].conclusion, 'neutral'); + } +}); + +test('missing, truncated or inconclusive replies get one scheduled retry after six hours', async () => { + for (const kind of ['missing', 'truncated-record', 'truncated-evidence', 'inconclusive']) { + const h = harness(); await h.run(); + if (kind !== 'missing') { + const reply = h.reply(kind === 'inconclusive' ? 'INCONCLUSIVE' : 'PASS'); + if (kind.startsWith('truncated')) { + const record = reply.body.match(/SEMANTIC_RESULT[^\n]+/)[0]; + const explanation = kind === 'truncated-record' ? 'coverage '.repeat(30) + record : record + ' evidence '.repeat(30); + reply.body = '\n' + + `| Semantic conflict with target branch | ✅ Passed | ${explanation.slice(0, 201)} |`; + } + } + h.setNow(NOW + 6 * HOUR - 1); await h.run('schedule'); + assert.equal(h.posted.length, 1, kind); + h.setNow(NOW + 6 * HOUR); await h.run(); assert.equal(h.posted.length, 1); + await h.run('schedule'); assert.equal(h.posted.length, 2, kind); + assert.equal(requests(h.comments)[0].retry, true); + assert.equal(h.checks.at(-1).conclusion, 'neutral'); + h.reply('INCONCLUSIVE'); + for (const elapsed of [12 * HOUR, DAY, 7 * DAY]) { + h.setNow(NOW + elapsed); await h.run('schedule'); + assert.equal(h.posted.length, 2, kind); + } + await h.run('workflow_dispatch'); assert.equal(h.posted.length, 3); + await h.publish(chatReply(h.reply())); assert.equal(h.checks.at(-1).conclusion, 'success'); + } +}); + +test('changed revisions are reassessed after an unavailable result and receive a fresh retry allowance', async () => { + const h = harness(); h.setLag(80, 3 * DAY); await h.run(); h.reply('INCONCLUSIVE'); + h.setNow(NOW + 6 * HOUR); await h.run('schedule'); assert.equal(h.posted.length, 2); + h.pr.head.sha = MERGED; h.setProgress(0); + h.setNow(NOW + DAY); await h.run('schedule'); assert.equal(h.posted.length, 2); + h.setNow(NOW + DAY + 6 * HOUR); await h.run('schedule'); assert.equal(h.posted.length, 3); + assert.equal(requests(h.comments)[0].retry, undefined); + h.setNow(NOW + DAY + 12 * HOUR); await h.run('schedule'); assert.equal(h.posted.length, 4); + assert.equal(requests(h.comments)[0].retry, true); + h.setNow(NOW + 3 * DAY); await h.run('schedule'); assert.equal(h.posted.length, 4); +}); + +test('30 new target commits allow a new version after an unavailable reply', async () => { + const h = harness(); await h.run(); h.reply('INCONCLUSIVE'); + h.setTarget(NEW_BASE); h.setNow(NOW + 2 * HOUR); h.setProgress(29); + await h.run('schedule'); assert.equal(h.posted.length, 1); + h.setProgress(30); await h.run('schedule'); assert.equal(h.posted.length, 2); +}); + +test('a valid reply arriving before the retry scan prevents a second request', async () => { + const h = harness(); await h.run(); h.setNow(NOW + 6 * HOUR); h.reply('FAIL'); + await h.run('schedule'); assert.equal(h.posted.length, 1); + assert.equal(h.checks[0].conclusion, 'failure'); +}); + +test('post-merge audits allow one retry without changing their fixed merged revisions', async () => { + const h = harness({merged: true, state: 'closed', merged_at: new Date(NOW).toISOString(), + updated_at: new Date(NOW).toISOString()}); + await h.run('pull_request_target', 'closed'); h.reply('INCONCLUSIVE'); + h.setTarget(NEW_BASE); h.setNow(NOW + 6 * HOUR); await h.run('schedule'); + assert.equal(h.posted.length, 2); + const pair = requests(h.comments)[0]; + assert.equal(pair.merged, MERGED); assert.equal(pair.target, BASE); assert.equal(pair.retry, true); + h.setNow(NOW + 12 * HOUR); await h.run('schedule'); assert.equal(h.posted.length, 2); +}); + +test('commands require the pinned User account and never fall back to the Actions bot', async () => { + for (const user of [ + {...COMMAND_USER, id: 1}, {...COMMAND_USER, login: 'another-user'}, + {...COMMAND_USER, type: 'Bot'}, {login: 'github-actions[bot]', type: 'Bot'}, + ]) { + const h = harness(); h.commandGithub.rest.users.getAuthenticated = async () => ({data: user}); + await assert.rejects(h.run('schedule'), /require the trtllm-agent/); + assert.equal(h.writes.length, 0); + } + const h = harness(); await h.run(); + assert.deepEqual(h.comments[0].user, COMMAND_USER); + assert.equal(requests(h.comments).length, 1); + h.comments[0].user.id = 1; assert.equal(requests(h.comments).length, 0); +}); + +test('scheduled scans cap new AI requests at 20 and later scans consider the remaining PRs', async () => { + const h = harness(); + for (let n = 13; n < 57; n++) h.prs.push({...h.pr, number: n}); + await h.run('schedule'); assert.equal(h.posted.length, 20); + assert.match(h.warnings.at(-1), /20 new AI requests/); + for (let round = 1; round <= 6; round++) { + const before = h.posted.length; + h.setNow(NOW + round * 6 * HOUR); await h.run('schedule'); + assert.ok(h.posted.length - before <= 20); + } + assert.equal(new Set(h.posted.map(p => p.issue_number)).size, 45); + for (const number of h.prs.map(p => p.number)) { + assert.ok(h.posted.filter(p => p.issue_number === number).length <= 2); + } + const beforeManual = h.posted.length; + await h.run('workflow_dispatch'); assert.equal(h.posted.length, beforeManual + 1); +}); + +test('scheduled scans prioritize recent merge recovery over open PR requests', async () => { + const h = harness(); + for (let n = 13; n < 40; n++) h.prs.push({...h.pr, number: n}); + h.prs.push({...h.pr, number: 100, state: 'closed', merged: true, + merged_at: new Date(NOW).toISOString(), updated_at: new Date(NOW).toISOString()}); + await h.run('schedule'); assert.equal(h.posted[0].issue_number, 100); + assert.match(h.posted[0].body, /Post-merge audit/); + assert.equal(h.posted.length, 20); +}); + +test('either token below its reserve stops scanning before PR reads; hooks are removed afterward', async () => { + for (const token of ['github', 'commandGithub']) { + const h = harness(); h[token].remaining = 99; await h.run('schedule'); + assert.equal(h.calls.length, 0); assert.equal(h.writes.length, 0); + assert.match(h.warnings.at(-1), /100-request reserve/); + await h.run('workflow_dispatch'); assert.equal(h.posted.length, 1); + } +}); + +test('response headers can stop a scan mid-PR before further pagination or writes', async () => { + const h = harness(); h.prs.push({...h.pr, number: 13}); + h.afterCompare(() => {h.github.remaining = 99;}); await h.run('schedule'); + assert.equal(h.calls.filter(([kind]) => kind === 'pull').length, 1); + assert.equal(h.calls.filter(([kind]) => kind === 'comments').length, 0); + assert.equal(h.writes.length, 0); assert.equal(h.failures.length, 0); + assert.match(h.warnings.at(-1), /100-request reserve/); +}); + +test('quota guard covers closed-list pagination and rotates open-PR starts', async () => { + const visited = []; + for (let round = 0; round < 2; round++) { + const h = harness(); h.prs.push({...h.pr, number: 13}); h.setNow(NOW + round * 6 * HOUR); + h.afterCompare(() => {h.github.remaining = 99;}); await h.run('schedule'); + visited.push(h.calls.find(([kind]) => kind === 'pull')[1].pull_number); + } + assert.notEqual(visited[0], visited[1]); + const h = harness(); let pages = 0; + h.github.paginate.iterator = async function* () { + await h.github.rest.pulls.get({pull_number: 12}); + h.github.remaining = 99; + await h.github.rest.pulls.get({pull_number: 12}); pages++; + yield {data: [{...h.pr, updated_at: new Date(NOW).toISOString()}]}; + await h.github.rest.pulls.get({pull_number: 12}); pages++; + yield {data: []}; + }; + await h.run('schedule'); assert.equal(pages, 1); assert.equal(h.writes.length, 0); +}); + +test('rate-limit exits stop the scan, while unrelated permission errors remain failures', async () => { + for (const error of [ + {status: 403, response: {headers: {'x-ratelimit-remaining': '0'}}}, + {status: 403, response: {headers: {'retry-after': '60'}}}, + {status: 403, message: 'You have exceeded a secondary rate limit'}, {status: 429}, + ]) { + const h = harness(); h.prs.push({...h.pr, number: 13}); + h.afterCompare(() => {h.github.apiError = Object.assign(new Error('Limited'), error);}); + await h.run('schedule'); assert.equal(h.writes.length, 0); assert.equal(h.failures.length, 0); + assert.equal(h.calls.filter(([kind]) => kind === 'pull').length, 1); + assert.match(h.warnings.at(-1), /Scheduled scan stopped/); + } + const h = harness(); h.prs.push({...h.pr, number: 13}); + h.afterCompare(() => {h.github.apiError = Object.assign(new Error('Resource not accessible'), {status: 403});}); + await h.run('schedule'); assert.ok(h.failures.length > 0); +}); diff --git a/.github/scripts/coderabbit_semantic_review_request.js b/.github/scripts/coderabbit_semantic_review_request.js new file mode 100644 index 000000000000..5adabf98aa30 --- /dev/null +++ b/.github/scripts/coderabbit_semantic_review_request.js @@ -0,0 +1,253 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require('node:fs'); +const publish = require('./coderabbit_semantic_review_result.js'); +const {NAME, AUDIT, NOTICE, supported, compare, requests, parseResult, matches, identity, evidence, candidateTree, isCommandUser} = publish; +const HOUR = 60 * 60 * 1000; +const DAY = 24 * HOUR; +const APPROVED = 'ci: full pre-merge approved'; + +function command(pair) { + const config = fs.readFileSync('.coderabbit.yaml', 'utf8'); + const match = config.match(/name: "Semantic conflict with target branch"[\s\S]*?instructions: \|\n((?: {10}[^\n]*(?:\n|$)|\n)*)/); + if (!match) throw new Error('Missing semantic check instructions'); + const instructions = match[1].replace(/^ {10}/gm, '').trim() + '\n' + + `Inspect these fixed revisions: head=${pair.head}, target=${pair.target}, merge_base=${pair.mergeBase}. ` + + (pair.merged ? `This is a post-merge audit. Also inspect the actual merged code at ${pair.merged}. ` + + `Include SEMANTIC_MERGED sha=${pair.merged} immediately after the result line. ` + + 'Later changes to the target branch do not invalidate this historical audit.' : + `This is a pre-merge analysis against ${pair.branch}. The result applies only to this pair.`); + return `@coderabbitai\nPlease perform this advisory semantic analysis and reply in a normal PR chat comment. ` + + `Do not invoke the custom pre-merge check command or submit a review/request changes.\n\n${instructions}`; +} + +async function requestOne({github, commandGithub, context, core, number, manual, now}) { + const repo = context.repo; + const {data: pr} = await github.rest.pulls.get({...repo, pull_number: number}); + if (!supported(pr.base.ref) || (!pr.merged && (pr.state !== 'open' || pr.draft))) { + if (manual) throw new Error(`PR #${number}: requires a non-draft open or merged main/release PR.`); + return; + } + const pair = await evidence(github, repo, pr); + const comments = await github.paginate(github.rest.issues.listComments, { + ...repo, issue_number: number, per_page: 100, + }); + const history = requests(comments).filter(r => r.branch === pair.branch); + const results = comments.toSorted((a, b) => b.id - a.id).map(parseResult).filter(Boolean); + const resultFor = r => results.find(v => matches(v, r) && v.comment.id > r.id && + Date.parse(v.comment.created_at) >= Date.parse(r.created_at)); + const reusable = history.find(r => r.head === pair.head && r.target === pair.target && + r.mergeBase === pair.mergeBase && (!pair.merged ? !r.merged : + (r.merged === pair.merged || (!r.merged && r.tree && r.tree === pair.tree)))); + const checks = await github.paginate(github.rest.checks.listForRef, { + ...repo, ref: pair.merged || pair.head, check_name: pair.merged ? AUDIT : NAME, + filter: 'all', per_page: 100, + }); + let check = checks.filter(c => c.app?.slug === 'github-actions' && + c.external_id?.startsWith(`semantic-v2:${number}:`)).sort((a, b) => b.id - a.id)[0]; + const currentId = identity(pr, pair); + // Clear a stale green/red result even if this revision is below the AI threshold. + if (check && check.external_id !== currentId && !pair.merged) { + await github.rest.checks.update({...repo, check_run_id: check.id, status: 'completed', + conclusion: 'neutral', output: {title: 'Previous semantic result is stale', + summary: `No current verdict for head ${pair.head} + ${pair.branch} ${pair.target}. ${NOTICE}`}}); + } + async function ensureCheck(reason) { + if (check?.external_id === currentId) return; + const output = {title: pair.merged ? 'Post-merge audit awaiting analysis' : + check ? 'Previous semantic result is stale' : 'No current AI verdict', + summary: `${reason} Head ${pair.head} + ${pair.branch} ${pair.target}. ${NOTICE}`}; + if (check) { + await github.rest.checks.update({...repo, check_run_id: check.id, external_id: currentId, + status: 'completed', conclusion: 'neutral', output}); + check.external_id = currentId; + } else { + ({data: check} = await github.rest.checks.create({...repo, + name: pair.merged ? AUDIT : NAME, head_sha: pair.merged || pair.head, + external_id: currentId, status: 'completed', conclusion: 'neutral', output})); + } + } + let retry = false; + if (reusable && !manual) { + const result = resultFor(reusable); + // Reuse an in-flight exact pair too. The publisher can complete the audit + // when its pre-merge reply arrives, provided the final tree also matches. + if (!pair.merged || reusable.merged || !result || result.verdict !== 'INCONCLUSIVE') { + retry = context.eventName === 'schedule' && !['PASS', 'FAIL'].includes(result?.verdict) && + now - Date.parse(reusable.created_at) >= 6 * HOUR && + !history.some(r => r.retry && matches(r, pair)); + if (!retry) { + await ensureCheck('This version already has an analysis request.'); + if (result) await publish({github, core, context: {...context, eventName: 'issue_comment', + payload: {issue: {number}}}}); + core.info(`PR #${number}: reusing the exact revision pair.`); + return; + } + } + } + const intent = context.eventName === 'pull_request_target' && + (context.payload.action === 'auto_merge_enabled' && pr.auto_merge || + context.payload.action === 'labeled' && context.payload.label?.name === APPROVED && + pr.labels.some(l => l.name === APPROVED) && process.env.SEMANTIC_APPROVAL_VALIDATED === 'true'); + let reason = manual ? 'Manual retry' : retry ? 'Automatic retry after unavailable analysis' : + pair.merged ? 'Post-merge audit' : intent ? 'Merge intent' : ''; + const last = history.find(r => !r.merged); + if (!reason) { + let count = pair.comparison.behind_by; + let since = Date.parse(pair.comparison.merge_base_commit.commit.committer.date); + let changed = count > 0; + if (last) { + const previous = resultFor(last); + const progress = await compare(github, repo, last.target, pair.target); + if (['ahead', 'identical'].includes(progress.status)) { + count = progress.ahead_by; + since = Date.parse(['PASS', 'FAIL'].includes(previous?.verdict) ? + previous.comment.created_at : last.created_at); + changed = last.head !== pair.head || last.target !== pair.target; + } + } + if (!changed || (count < 30 && now - since < DAY)) { + await ensureCheck('Below the 24-hour / 30-commit threshold; waiting for changed revisions to qualify.'); + return; + } + reason = '24-hour / 30-commit threshold'; + } + // Approval and auto-merge share this budget even when the branch SHAs change. + // Count any recent pre-merge request, so a routine scan cannot double the cost. + if (!manual && !pair.merged && last && now - Date.parse(last.created_at) < HOUR) { + await ensureCheck('Pre-merge analysis is in its one-hour cooldown; old results are stale.'); + return; + } + const {data: current} = await github.rest.pulls.get({...repo, pull_number: number}); + if (current.head.sha !== pair.head || current.base.ref !== pair.branch || + current.state !== pr.state || current.merged !== pr.merged || current.draft !== pr.draft || + (pair.merged && current.merge_commit_sha !== pair.merged)) { + if (manual) throw new Error('Manual retry not posted: PR changed during validation.'); + return; + } + if (!pair.merged) { + const {data: ref} = await github.rest.git.getRef({...repo, ref: `heads/${pair.branch}`}); + if (ref.object.sha !== pair.target) { + if (manual) throw new Error('Manual retry not posted: target changed during validation.'); + return; + } + } + await ensureCheck(reason); + await github.rest.checks.update({...repo, check_run_id: check.id, + status: 'completed', conclusion: 'neutral', + output: {title: 'Awaiting CodeRabbit analysis', + summary: `${reason}: head ${pair.head}, target ${pair.target}. ${NOTICE}`}}); + if (!pair.merged) pair.tree = await candidateTree(github, repo, current, pair.target); + const {comparison, ...record} = pair; + const {data: comment} = await commandGithub.rest.issues.createComment({...repo, issue_number: number, + body: `${command(pair)}\n\n\n\n` + + `${reason} for PR #${number}. ${NOTICE} No AI verdict is asserted by posting this request.`}); + core.info(`PR #${number}: ${reason}; ${comment.html_url}`); + await core.summary.addRaw(`PR #${number}: [${reason}](${comment.html_url}). No AI verdict is asserted.\n\n`).write(); + return true; +} + +class ScanStopped extends Error {} + +function rateLimited(error) { + const headers = error.response?.headers || {}; + return error.status === 429 || error.status === 403 && + (headers['x-ratelimit-remaining'] === '0' || headers['retry-after'] !== undefined || + /rate limit|abuse detection/i.test(error.message)); +} + +async function run({github, commandGithub, context, core, now, progress}) { + let numbers; + const manual = context.eventName === 'workflow_dispatch'; + if (manual) { + const raw = (process.env.DISPATCH_PULL_NUMBER || '').trim(); + const number = Number(raw); + if (!/^[1-9][0-9]*$/.test(raw) || !Number.isSafeInteger(number)) { + throw new Error('pull_number must be a positive integer'); + } + numbers = [number]; + } else if (context.eventName === 'pull_request_target') { + numbers = [context.payload.pull_request.number]; + } else if (context.eventName === 'schedule') { + const pulls = await github.paginate(github.rest.pulls.list, { + ...context.repo, state: 'open', per_page: 100, + }); + const open = pulls.filter(p => !p.draft && supported(p.base.ref)).map(p => p.number).sort((a, b) => a - b); + // Rotate the starting PR each scan; no persistent cursor or exact resume is implied. + const offset = Math.floor(now / (6 * HOUR)) % (open.length || 1); + numbers = []; + // Recover recent merges if an event was dropped or a release branch still + // lacks the workflow. Do not backfill the repository's entire merge history. + for await (const response of github.paginate.iterator(github.rest.pulls.list, { + ...context.repo, state: 'closed', sort: 'updated', direction: 'desc', per_page: 100, + })) { + numbers.push(...response.data.filter(p => supported(p.base.ref) && + Date.parse(p.merged_at) >= now - DAY).map(p => p.number)); + if (!response.data.length || Date.parse(response.data.at(-1).updated_at) < now - DAY) break; + } + numbers = [...new Set([...numbers, ...open.slice(offset), ...open.slice(0, offset)])]; + } else throw new Error(`Unsupported event: ${context.eventName}`); + for (const number of numbers) { + if (context.eventName === 'schedule' && progress.requested >= 20) { + throw new ScanStopped('Reached the limit of 20 new AI requests for this scan.'); + } + try { + if (await requestOne({github, commandGithub, context, core, number, manual, now})) progress.requested++; + progress.processed++; + } catch (error) { + if (context.eventName !== 'schedule' || error instanceof ScanStopped || rateLimited(error)) throw error; + core.error(`PR #${number}: ${error.message}`); + core.setFailed('Some PRs could not be scanned; see per-PR errors.'); + } + } +} + +module.exports = async ({github, commandGithub, context, core, now = Date.now()}) => { + const scheduled = context.eventName === 'schedule'; + const progress = {processed: 0, requested: 0}; + const guards = []; + let stopReason = 'Complete'; + try { + if (scheduled) { + for (const client of [github, commandGithub]) { + let remaining = Infinity; + const guard = async (request, options) => { + if (remaining < 100) throw new ScanStopped('REST quota is below the 100-request reserve.'); + const response = await request(options); + const value = response.headers?.['x-ratelimit-remaining']; + if (value !== undefined) remaining = Number(value); + return response; + }; + client.hook.wrap('request', guard); + guards.push([client, guard]); + const {data} = await client.rest.rateLimit.get(); + remaining = data.resources.core.remaining; + } + } + const {data: user} = await commandGithub.rest.users.getAuthenticated(); + if (!isCommandUser(user)) throw new Error('Semantic commands require the trtllm-agent User account.'); + await run({github, commandGithub, context, core, now, progress}); + } catch (error) { + if (!scheduled || !(error instanceof ScanStopped || rateLimited(error))) throw error; + stopReason = error.message; + core.warning(`Scheduled scan stopped: ${stopReason} Unvisited PRs wait for a later scan or manual dispatch.`); + } finally { + for (const [client, guard] of guards) client.hook.remove('request', guard); + if (scheduled) await core.summary.addRaw(`Scheduled scan: ${progress.processed} PRs processed; ` + + `${progress.requested} new AI requests. ${stopReason}. Scans restart from live state, not a saved cursor.\n`).write(); + } +}; +Object.assign(module.exports, {command}); diff --git a/.github/scripts/coderabbit_semantic_review_result.js b/.github/scripts/coderabbit_semantic_review_result.js new file mode 100644 index 000000000000..99650789ca48 --- /dev/null +++ b/.github/scripts/coderabbit_semantic_review_result.js @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const NAME = 'Semantic conflict with target branch'; +const AUDIT = 'Semantic conflict audit (post-merge)'; +const NOTICE = 'CodeRabbit can make mistakes, including false positives. Review the evidence. ' + + 'This check is advisory and must remain non-required; its failure does not block merging ' + + 'under that configuration. Other merge requirements still apply.'; +const supported = ref => ref === 'main' || /^release\/.+/.test(ref); +const isBot = (comment, login) => comment.user?.login === login && comment.user?.type === 'Bot'; +// Pin the existing service account by ID as well as login; never trust arbitrary PAT users. +const isCommandUser = user => user?.login === 'trtllm-agent' && user.id === 296075020 && user.type === 'User'; +const compare = async (github, repo, base, head) => (await github.request( + 'GET /repos/{owner}/{repo}/compare/{basehead}', {...repo, basehead: `${base}...${head}`} +)).data; + +// Request metadata is written only by the trusted workflow, never accepted from PR authors. +function requests(comments) { + return comments.filter(c => isCommandUser(c.user) || isBot(c, 'github-actions[bot]')).flatMap(c => { + const text = c.body?.match(//); + if (!text) return []; + try { + const request = JSON.parse(text[1]); + if (![request.head, request.target, request.mergeBase].every(s => /^[a-f0-9]{40}$/.test(s)) || + !supported(request.branch)) return []; + return [{...request, created_at: c.created_at, id: c.id}]; + } catch { return []; } + }).sort((a, b) => b.id - a.id); +} + +function parseResult(comment) { + const body = comment.body || ''; + if (!isBot(comment, 'coderabbitai[bot]')) return; + // Chat replies retain full evidence for PASS as well as FAIL. Native custom + // checks may truncate PASS to a 201-character cell; keep old replies readable. + const standalone = body.match(/^SEMANTIC_REVIEW_V3[ \t]*\r?$/m); + if (!standalone && (!body.includes('') && + !body.includes(''))) return; + const row = body.split('\n').map(line => line.split('|').map(cell => cell.trim())) + .find(cells => cells[1]?.toLowerCase() === NAME.toLowerCase()); + if (!standalone && !row) return; + const details = body.match(/Full details: Semantic conflict with target branch<\/summary>([\s\S]*?)<\/details>/i); + const result = standalone ? body.slice(standalone.index + standalone[0].length) : details ? details[1] : row.join('|'); + const pattern = /semantic_result head=([a-f0-9]{40}) target=([a-f0-9]{40}) merge_base=([a-f0-9]{40}) verdict=(pass|fail|inconclusive)\b/g; + const matches = [...new Map([...result.toLowerCase().matchAll(pattern)].map(m => [m[0], m])).values()]; + if (matches.length !== 1) return; + const [, head, target, mergeBase, rawVerdict] = matches[0]; + let verdict = rawVerdict.toUpperCase(); + const status = {PASS: /Passed/i, FAIL: /Warning|Error/i, INCONCLUSIVE: /Inconclusive/i}; + if (!standalone && !status[verdict].test(row[2])) return; + const merged = [...new Set([...result.toLowerCase().matchAll(/semantic_merged sha=([a-f0-9]{40})\b/g)].map(m => m[1]))]; + if (merged.length > 1) return; + // This checks citation presence, not the correctness of the AI's reasoning. + const repository = comment.html_url?.match(/^https:\/\/github\.com\/([^/]+\/[^/]+)\/pull\//i)?.[1].toLowerCase(); + const citations = [...result.matchAll(/https:\/\/github\.com\/([^/\s]+\/[^/\s]+)\/blob\/([a-f0-9]{40})\/[^\s<>)|]+#L[1-9]\d*/gi)] + .filter(m => m[1].toLowerCase() === repository).map(m => m[2].toLowerCase()); + const missingEvidence = verdict !== 'INCONCLUSIVE' && ![head, target].every(sha => citations.includes(sha)); + if (missingEvidence) verdict = 'INCONCLUSIVE'; + return {head, target, mergeBase, verdict, missingEvidence, merged: merged[0], comment}; +} + +const matches = (result, request) => result && result.head === request.head && + result.target === request.target && result.mergeBase === request.mergeBase && + result.merged === request.merged; +const identity = (pr, pair) => `semantic-v2:${pr.number}:${pair.head}:${pair.target}:${pair.merged || 'open'}`; + +async function evidence(github, repo, pr) { + let target; + let tree; + let merged; + if (pr.merged) { + merged = pr.merge_commit_sha; + const {data: commit} = await github.rest.git.getCommit({...repo, commit_sha: merged}); + // The repository requires squash merges. A normal two-parent merge is also + // unambiguous; rebases must not silently be treated as a squash merge. + if (commit.parents.length === 1) { + const {data: rules} = await github.request('GET /repos/{owner}/{repo}/rules/branches/{branch}', + {...repo, branch: pr.base.ref}); + if (!rules.some(r => r.type === 'pull_request' && + r.parameters?.allowed_merge_methods?.length === 1 && + r.parameters.allowed_merge_methods[0] === 'squash')) { + throw new Error('Cannot verify a squash-only merge policy; historical target is inconclusive.'); + } + } else if (commit.parents.length !== 2 || commit.parents[1].sha !== pr.head.sha) { + throw new Error('Cannot verify the historical merge parents.'); + } + target = commit.parents[0].sha; + tree = commit.tree.sha; + } else { + const {data: ref} = await github.rest.git.getRef({...repo, ref: `heads/${pr.base.ref}`}); + target = ref.object.sha; + + } + const comparison = await compare(github, repo, target, pr.head.sha); + return {head: pr.head.sha, target, mergeBase: comparison.merge_base_commit.sha, + branch: pr.base.ref, merged, tree, comparison}; +} + +// This extra API read is needed only when actually requesting AI, not on every scan. +async function candidateTree(github, repo, pr, target) { + if (!pr.merge_commit_sha) return; + try { + const {data: candidate} = await github.rest.git.getCommit({...repo, commit_sha: pr.merge_commit_sha}); + if (candidate.parents.length === 2 && candidate.parents[0].sha === target && + candidate.parents[1].sha === pr.head.sha) return candidate.tree.sha; + } catch (error) { + if (error.status !== 404 && error.status !== 409) throw error; + } +} + +// Both the privileged publisher and the read-only PR preview use this verifier. +async function publish({github, context, core}) { + const preview = context.eventName === 'pull_request'; + if (preview) core.setOutput('verdict', 'INCONCLUSIVE'); + const repo = context.repo; + const number = preview ? context.payload.pull_request.number : context.payload.issue.number; + const {data: pr} = await github.rest.pulls.get({...repo, pull_number: number}); + if (!supported(pr.base.ref) || (!pr.merged && (pr.state !== 'open' || (!preview && pr.draft)))) return; + if (preview && (pr.merged || pr.head.sha !== context.payload.pull_request.head.sha)) { + core.warning('This preview is stale; use a run for the current PR head.'); + return; + } + const pair = await evidence(github, repo, pr); + const comments = await github.paginate(github.rest.issues.listComments, { + ...repo, issue_number: number, per_page: 100, + }); + const allRequests = requests(comments); + const latestRequest = allRequests.find(r => r.branch === pair.branch && + r.head === pair.head && r.target === pair.target && r.mergeBase === pair.mergeBase && + (pair.merged ? (r.merged === pair.merged || (!r.merged && r.tree && r.tree === pair.tree)) : !r.merged)); + let result; + // Reconcile the latest API result, even when an older comment event is replayed. + for (const comment of comments.toSorted((a, b) => b.id - a.id)) { + const parsed = parseResult(comment); + if (!parsed || parsed.head !== pair.head || parsed.target !== pair.target || + parsed.mergeBase !== pair.mergeBase) continue; + if (preview && !latestRequest && !parsed.merged) { result = parsed; break; } + if (latestRequest && matches(parsed, latestRequest) && + comment.id > latestRequest.id && + Date.parse(comment.created_at) >= Date.parse(latestRequest.created_at)) { + result = parsed; break; + } + } + if (!result) { + if (preview) { + const message = `No verified CodeRabbit verdict for head ${pair.head} + ${pair.branch} ${pair.target}. ` + + 'Request a semantic evaluation and rerun the tests and result lookup job after the reply arrives.'; + core.warning(message); + await core.summary.addRaw(`${message}\n\n${NOTICE}`).write(); + } + return; + } + const {data: current} = await github.rest.pulls.get({...repo, pull_number: number}); + if (current.head.sha !== pair.head || current.base.ref !== pair.branch || + current.merged !== pr.merged || current.state !== pr.state || (!preview && current.draft)) return; + if (pair.merged) { + if (current.merge_commit_sha !== pair.merged) return; + } else { + const {data: ref} = await github.rest.git.getRef({...repo, ref: `heads/${pair.branch}`}); + if (ref.object.sha !== pair.target) { + if (preview) core.warning('The target changed during this preview; the result is stale.'); + return; + } + } + const title = {PASS: 'No semantic conflict found (best effort)', + FAIL: 'Possible semantic conflict — CodeRabbit may be wrong', + INCONCLUSIVE: 'Semantic analysis inconclusive'}[result.verdict]; + const summary = `${pair.merged ? `Post-merge audit of ${pair.merged}. ` : ''}` + + `Verified head ${pair.head}, target ${pair.target}, merge base ${pair.mergeBase}.\n\n` + + (result.missingEvidence ? 'CodeRabbit did not provide source links with full SHAs and line numbers for both revisions; ' + + 'its reported verdict is treated as inconclusive.\n\n' : '') + + `[CodeRabbit analysis](${result.comment.html_url})\n\n${NOTICE}`; + if (!preview) { + const checks = await github.paginate(github.rest.checks.listForRef, { + ...repo, ref: pair.merged || pair.head, check_name: pair.merged ? AUDIT : NAME, + filter: 'all', per_page: 100, + }); + const check = checks.filter(c => c.app?.slug === 'github-actions' && c.external_id === identity(pr, pair)) + .sort((a, b) => b.id - a.id)[0]; + if (!check) return; + await github.rest.checks.update({...repo, check_run_id: check.id, status: 'completed', + conclusion: {PASS: 'success', FAIL: 'failure', INCONCLUSIVE: 'neutral'}[result.verdict], + details_url: result.comment.html_url, output: {title, summary}}); + if (pair.merged) { + const marker = ``; + if (!comments.some(c => isBot(c, 'github-actions[bot]') && c.body?.includes(marker))) { + await github.rest.issues.createComment({...repo, issue_number: number, + body: `${marker}\n**Post-merge semantic audit: ${result.verdict}**\n\n${summary}`}); + } + } + } + await core.summary.addRaw(`${preview ? 'Read-only PR preview\n\n' : ''}${title}\n\n${summary}\n`).write(); + core.info(`Verified semantic verdict: ${result.verdict}; head=${pair.head}; target=${pair.target}`); + const message = `${title}. ${NOTICE} ${result.comment.html_url}`; + if (preview) { + core.setOutput('verdict', result.verdict); + core.setOutput('summary', `${title}\n\n${summary}`); + core.setOutput('message', message); + } else if (result.verdict === 'FAIL') core.setFailed(message); + if (result.verdict === 'INCONCLUSIVE') core.warning(message); +} + +module.exports = publish; +Object.assign(module.exports, {NAME, AUDIT, NOTICE, supported, compare, requests, parseResult, matches, identity, evidence, candidateTree, isCommandUser}); diff --git a/.github/workflows/coderabbit-semantic-review-tests.yml b/.github/workflows/coderabbit-semantic-review-tests.yml new file mode 100644 index 000000000000..76ec3beaa71b --- /dev/null +++ b/.github/workflows/coderabbit-semantic-review-tests.yml @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CodeRabbit Semantic Review Preview + +on: + pull_request: + branches: [main, 'release/**'] + paths: + - '.coderabbit.yaml' + - '.github/workflows/coderabbit-semantic-review*.yml' + - '.github/scripts/coderabbit_semantic_review*.js' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Automation tests and result lookup (not AI approval) + permissions: + contents: read + pull-requests: read + issues: read + outputs: + verdict: ${{ steps.result.outputs.verdict }} + summary: ${{ steps.result.outputs.summary }} + message: ${{ steps.result.outputs.message }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - name: Test semantic review automation + run: node --test .github/scripts/coderabbit_semantic_review.test.js + - name: Read CodeRabbit verdict for the current PR and target branch + id: result + if: github.event_name == 'pull_request' + uses: actions/github-script@v8 + with: + script: | + const readResult = require('./.github/scripts/coderabbit_semantic_review_result.js'); + await readResult({github, context, core}); + + verdict: + name: AI verdict (advisory; skipped = unavailable) + needs: test + if: >- + github.event_name == 'pull_request' && + contains(fromJSON('["PASS", "FAIL"]'), needs.test.outputs.verdict) + permissions: {} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Display the verified AI verdict + uses: actions/github-script@v8 + env: + SEMANTIC_VERDICT: ${{ needs.test.outputs.verdict }} + SEMANTIC_SUMMARY: ${{ needs.test.outputs.summary }} + SEMANTIC_MESSAGE: ${{ needs.test.outputs.message }} + with: + script: | + await core.summary.addRaw(process.env.SEMANTIC_SUMMARY).write(); + if (process.env.SEMANTIC_VERDICT === 'FAIL') { + core.setFailed(process.env.SEMANTIC_MESSAGE); + } diff --git a/.github/workflows/coderabbit-semantic-review.yml b/.github/workflows/coderabbit-semantic-review.yml new file mode 100644 index 000000000000..778717bcf1f8 --- /dev/null +++ b/.github/workflows/coderabbit-semantic-review.yml @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: CodeRabbit Semantic Conflict Review + +on: + pull_request_target: + branches: [main, 'release/**'] + types: [opened, reopened, synchronize, ready_for_review, labeled, edited, auto_merge_enabled, closed] + schedule: + - cron: '23 */6 * * *' + issue_comment: + types: [created, edited] + workflow_dispatch: + inputs: + pull_number: + description: 'Main/release PR to recheck; merged PRs receive a post-merge audit' + required: true + type: string + +permissions: + contents: read + +jobs: + request-review: + name: Request advisory semantic review or audit + permissions: + contents: read + pull-requests: read + issues: write + checks: write + if: >- + github.repository == 'NVIDIA/TensorRT-LLM' && + contains(fromJSON('["schedule", "pull_request_target", "workflow_dispatch"]'), github.event_name) && + (github.event.action != 'closed' || github.event.pull_request.merged) && + (github.event.action != 'edited' || github.event.changes.base.ref) && + (github.event.action != 'labeled' || github.event.label.name == 'ci: full pre-merge approved') + # Requests and publications share a queue: scheduled scans cannot race PR events. + # ponytail: one queue, 100 pending; use per-PR dispatch if scans delay events. + concurrency: + group: coderabbit-semantic-state + queue: max + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + with: + # Always execute the default branch's trusted code, including release events. + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + sparse-checkout: | + /.coderabbit.yaml + /.github/scripts/coderabbit_semantic_review*.js + sparse-checkout-cone-mode: false + - name: Verify approval-label author using the existing approver team + id: approval + if: github.event.action == 'labeled' + uses: actions/github-script@v8 + with: + github-token: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }} + result-encoding: string + script: | + const {data: pr} = await github.rest.pulls.get({ + ...context.repo, pull_number: context.payload.pull_request.number, + }); + if (!pr.labels.some(l => l.name === 'ci: full pre-merge approved')) return false; + const events = await github.paginate(github.rest.issues.listEventsForTimeline, { + ...context.repo, issue_number: pr.number, per_page: 100, + }); + const latest = events.filter(e => e.event === 'labeled' && + e.label?.name === 'ci: full pre-merge approved').at(-1); + if (!latest?.actor?.login || latest.actor.login !== context.payload.sender?.login) return false; + try { + const {data} = await github.rest.teams.getMembershipForUser({ + org: 'NVIDIA', team_slug: 'trt-llm-ci-approvers', username: latest.actor.login, + }); + return data.state === 'active'; + } catch (error) { + if (![403, 404].includes(error.status)) throw error; + core.warning('The approval-label author could not be verified; no immediate AI request.'); + return false; + } + - name: Apply the threshold, cooldown and audit policy + if: github.event.action != 'labeled' || steps.approval.outputs.result == 'true' + uses: actions/github-script@v8 + env: + DISPATCH_PULL_NUMBER: ${{ inputs.pull_number }} + SEMANTIC_APPROVAL_VALIDATED: ${{ steps.approval.outputs.result }} + SEMANTIC_COMMAND_TOKEN: ${{ secrets.TRTLLM_AGENT_SHARED_TOKEN }} + with: + script: | + const request = require('./.github/scripts/coderabbit_semantic_review_request.js'); + if (!process.env.SEMANTIC_COMMAND_TOKEN) throw new Error('Missing semantic command token.'); + const commandGithub = new github.constructor({ + auth: process.env.SEMANTIC_COMMAND_TOKEN, retry: {enabled: false}, + }); + await request({github, commandGithub, context, core}); + + publish-result: + name: Publish advisory semantic result + permissions: + contents: read + pull-requests: read + issues: write + checks: write + if: >- + github.repository == 'NVIDIA/TensorRT-LLM' && + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.user.login == 'coderabbitai[bot]' && + contains(github.event.comment.body, 'SEMANTIC_REVIEW_V3') + concurrency: + group: coderabbit-semantic-state + queue: max + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + sparse-checkout: .github/scripts/coderabbit_semantic_review_result.js + sparse-checkout-cone-mode: false + - name: Verify the requested revisions and publish the advisory verdict + uses: actions/github-script@v8 + with: + script: | + const publish = require('./.github/scripts/coderabbit_semantic_review_result.js'); + await publish({github, context, core}); diff --git a/AGENTS.md b/AGENTS.md index bcbb5cf10368..d5c11dd91ba3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -175,6 +175,11 @@ CI is triggered by posting comments on the PR. Basic commands: For a full list of up-to-date bot commands, post `/bot help` as a PR comment and check the bot's reply. +### Advisory semantic conflict review + +For trigger rules, retries and result verification, see the +[semantic review operator guide](.github/coderabbit-semantic-review.md). + ### Trouble Shooting - Use `TLLM_LOG_LEVEL_BY_MODULE` to enable per-module log filtering (e.g., `"debug:_torch,runtime;info:serve"`); see [Module-Level Logging](docs/source/developer-guide/overview.md#module-level-logging) for details.