From f23e4f321aafd079f007827551e9449a0f82fa6f Mon Sep 17 00:00:00 2001 From: ZhenghuaBao Date: Fri, 24 Jul 2026 11:22:17 +0800 Subject: [PATCH 1/7] Address review follow-ups: threshold/id validation, rehome line numbers, non-code path polarity, uncovered fail-open, policy-block cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit judge.mjs: - Validate --threshold up front (reject NaN / out-of-range so a typo does not silently drop every finding via `x >= NaN`). - Bump the uncovered fail-open confidence from 0.5 to 1.0 so raising --threshold above 0.5 does not turn the fail-open path into a fail-closed one that discards the judge's blind spots. - Guard the kept-set push against a malformed judge group whose representative_id resolves to undefined (would otherwise inject an undefined comment and crash downstream). postfilter.mjs: - Use `git grep -n` and carry the matched line number into rehomed findings so the posted comment lands on the actual snippet rather than the reviewer's original (now-wrong) line after a rehome. - Flip the "unknown extension → drop" polarity to "known-non-code extension → drop" so Dockerfile, Makefile, and other extensionless code files no longer get dropped as if they were locale JSON. action.yml: - Clear $POLICY_BLOCK when the L2 judge soft-fails, so a guardrail block that hit only the sidecar judge call is not surfaced as if it blocked the engine's review. --- action.yml | 6 +++++ scripts/judge.mjs | 43 ++++++++++++++++++++++++------ scripts/postfilter.mjs | 59 +++++++++++++++++++++++++++++++++--------- 3 files changed, 88 insertions(+), 20 deletions(-) diff --git a/action.yml b/action.yml index 2f86f13..f6b7713 100644 --- a/action.yml +++ b/action.yml @@ -829,6 +829,12 @@ runs: mv "$L2_OUT" "$1" else echo "::warning::L2 judge failed — keeping L1 findings" + # If a guardrail/firewall blocked the judge request the proxy + # will have written $POLICY_BLOCK — but that block belongs to + # the L2 sidecar call, NOT to the engine's own review. Drop + # the file so the downstream "Surface guardrail block" step + # does not surface a spurious block from a soft-failed judge. + rm -f "$POLICY_BLOCK" fi echo "::endgroup::" fi diff --git a/scripts/judge.mjs b/scripts/judge.mjs index f6031d1..1fa27cf 100644 --- a/scripts/judge.mjs +++ b/scripts/judge.mjs @@ -21,7 +21,18 @@ const [file, ...rest] = process.argv.slice(2); let out = null, threshold = 0.7, modelOverride = null; for (let i = 0; i < rest.length; i += 1) { if (rest[i] === "--out") out = rest[++i]; - else if (rest[i] === "--threshold") threshold = parseFloat(rest[++i]); + else if (rest[i] === "--threshold") { + // Validate the numeric range up front so a typo (or an unset env + // interpolation resolving to empty string) fails loudly instead of + // silently producing NaN, which — via `x >= NaN` = false — would + // drop every finding and emit an empty kept set. + const rawT = rest[++i]; + threshold = parseFloat(rawT); + if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + console.error(`judge: --threshold must be a number in [0,1], got ${JSON.stringify(rawT)}`); + process.exit(2); + } + } else if (rest[i] === "--model") modelOverride = rest[++i]; } if (!file) { console.error("usage: node judge.mjs [--out f] [--threshold 0.7] [--model deepseek/deepseek-v4-pro]"); process.exit(2); } @@ -112,18 +123,34 @@ catch (e) { console.error("judge did not return JSON:\n" + content.slice(0, 600) const groups = parsed.groups || []; const covered = new Set(); for (const g of groups) for (const id of g.member_ids || []) covered.add(id); +// Fail-open for findings the judge did not classify into any group: mark +// them keep with a synthetic confidence that survives ANY user-set +// threshold, so raising the threshold does not silently drop the judge's +// blind spots (the previous 0.5 constant fails closed at threshold > 0.5, +// contradicting the "fail-open" comment). for (let i = 0; i < findings.length; i += 1) if (!covered.has(i)) - groups.push({ member_ids: [i], representative_id: i, confidence: 0.5, keep: true, root_cause: "(uncovered)", reason: "not classified by judge; kept fail-open" }); + groups.push({ member_ids: [i], representative_id: i, confidence: 1.0, keep: true, root_cause: "(uncovered)", reason: "not classified by judge; kept fail-open above any threshold" }); const kept = []; const dropped = []; for (const g of groups) { - const surv = g.keep && g.confidence >= threshold; - const rep = comments[g.representative_id] ?? comments[g.member_ids[0]]; - const others = (g.member_ids || []).filter((id) => id !== (g.representative_id ?? g.member_ids[0])); - const line = `[conf ${g.confidence?.toFixed(2)}] ${(rep?.content || "").match(/\[(P[0-3])\]/)?.[0] || ""} ${rep?.path} :: ${g.root_cause} ${others.length ? "(merged " + others.length + ")" : ""}`; - if (surv) { kept.push(rep); console.error("KEEP " + line); } - else { dropped.push(g); console.error("drop " + line + " — " + (g.reason || "")); } + const surv = g.keep && Number.isFinite(g.confidence) && g.confidence >= threshold; + // Resolve the representative comment through both the primary id and the + // first-member fallback, but only push if we actually land on a real + // comment — a malformed judge group with an out-of-range id would + // otherwise inject `undefined` into the kept set and crash downstream. + const memberFallback = Array.isArray(g.member_ids) && g.member_ids.length ? g.member_ids[0] : null; + const repId = comments[g.representative_id] != null ? g.representative_id : memberFallback; + const rep = repId != null ? comments[repId] : null; + const others = Array.isArray(g.member_ids) + ? g.member_ids.filter((id) => id !== repId) + : []; + const line = `[conf ${g.confidence?.toFixed?.(2) ?? "?"}] ${(rep?.content || "").match(/\[(P[0-3])\]/)?.[0] || ""} ${rep?.path ?? "?"} :: ${g.root_cause} ${others.length ? "(merged " + others.length + ")" : ""}`; + if (surv) { + if (!rep) { console.error("skip malformed judge group (no valid rep): " + JSON.stringify(g).slice(0, 120)); continue; } + kept.push(rep); + console.error("KEEP " + line); + } else { dropped.push(g); console.error("drop " + line + " — " + (g.reason || "")); } } if (out) fs.writeFileSync(out, JSON.stringify({ ...data, comments: kept }, null, 1)); diff --git a/scripts/postfilter.mjs b/scripts/postfilter.mjs index 2d415c9..b3bbaee 100644 --- a/scripts/postfilter.mjs +++ b/scripts/postfilter.mjs @@ -24,16 +24,40 @@ if (!file || !repo || !commit) { process.exit(2); } -const CODE_EXT = /\.(go|js|jsx|ts|tsx|mjs|cjs|py|java|rb|rs|c|h|cc|cpp|sql|sh)$/i; +// Only drop findings whose snippet did NOT match anywhere in the tree when +// the claimed path is a KNOWN non-code file (locale JSON / doc markdown / +// changelogs / lockfiles). Extensionless code files (Dockerfile, Makefile, +// scripts with no extension) previously fell through the "not code-ext" +// branch and got dropped as if they were locale files — reverse the polarity +// so only clearly-non-code paths incur the drop. +const KNOWN_NON_CODE = /\.(json|ya?ml|md|txt|lock|log|csv|tsv|po|pot|properties)$/i; +// Returns an array of { file, line } — each match of `pattern` at the given +// commit, with the line NUMBER as reported by git-grep -n so a rehome can +// point at the actual snippet's line rather than reusing the reviewer's +// (now-wrong) original line. function grepFiles(pattern) { if (!pattern || pattern.length < 12) return []; try { - const o = execFileSync("git", ["-C", repo, "grep", "-F", "-I", "-l", "-e", pattern, commit], { + const o = execFileSync("git", ["-C", repo, "grep", "-F", "-I", "-n", "-e", pattern, commit], { encoding: "utf8", maxBuffer: 1 << 24, }); - return [...new Set(o.split("\n").filter(Boolean).map((l) => l.slice(l.indexOf(":") + 1)))]; + // Output shape: ":::". Strip the commit prefix + // once (present because we asked for a specific commit), then parse. + const seen = new Map(); // file -> first line seen (dedup multi-hit files) + for (const raw of o.split("\n")) { + if (!raw) continue; + const afterCommit = raw.slice(raw.indexOf(":") + 1); // "::" + const firstColon = afterCommit.indexOf(":"); + if (firstColon < 0) continue; + const file = afterCommit.slice(0, firstColon); + const rest = afterCommit.slice(firstColon + 1); + const secondColon = rest.indexOf(":"); + const line = secondColon >= 0 ? Number(rest.slice(0, secondColon)) : null; + if (!seen.has(file)) seen.set(file, Number.isFinite(line) ? line : null); + } + return [...seen.entries()].map(([file, line]) => ({ file, line })); } catch { return []; // grep exit 1 = no match } @@ -54,25 +78,36 @@ const report = []; const kept = []; for (const c of comments) { - let trueFiles = []; + let hits = []; // [{file, line}] for (const ln of candidateLines(c.existing_code)) { const f = grepFiles(ln); - if (f.length) { trueFiles = f; break; } + if (f.length) { hits = f; break; } } let path = c.path; + let start_line = c.start_line; + let end_line = c.end_line; let action = "keep"; - if (trueFiles.length) { - if (trueFiles.includes(c.path)) action = "keep (correct)"; - else if (trueFiles.length === 1) { path = trueFiles[0]; action = `REHOME ${c.path} -> ${path}`; } - else action = `keep (ambiguous: ${trueFiles.length} files)`; - } else if (!CODE_EXT.test(c.path)) { - action = "DROP (code snippet, filed on non-code file, not found)"; + const files = hits.map((h) => h.file); + if (hits.length) { + if (files.includes(c.path)) action = "keep (correct)"; + else if (hits.length === 1) { + // REHOME: the finding is really about `hits[0].file`, not the claimed + // path — update BOTH path and lines so the posted comment lands on + // the actual snippet, not the reviewer's original (unrelated) line. + const hit = hits[0]; + path = hit.file; + if (Number.isFinite(hit.line)) { start_line = hit.line; end_line = hit.line; } + action = `REHOME ${c.path} -> ${path}${Number.isFinite(hit.line) ? ":" + hit.line : ""}`; + } + else action = `keep (ambiguous: ${hits.length} files)`; + } else if (KNOWN_NON_CODE.test(c.path)) { + action = "DROP (code snippet, filed on known-non-code file, not found)"; } else { action = "keep (snippet unverified)"; } report.push({ sev: (c.content || "").match(/\[(P[0-3])\]/)?.[1] || "?", from: c.path, action }); if (action.startsWith("DROP")) continue; - kept.push({ ...c, path }); + kept.push({ ...c, path, start_line, end_line }); } const norm = (s) => From 0f88e8571b669c6ddaeab11fc1a7b23c59811ee0 Mon Sep 17 00:00:00 2001 From: ZhenghuaBao Date: Fri, 24 Jul 2026 11:54:23 +0800 Subject: [PATCH 2/7] docs(judge): keep max_tokens rationale comment generic --- scripts/judge.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/judge.mjs b/scripts/judge.mjs index 1fa27cf..ecc5323 100644 --- a/scripts/judge.mjs +++ b/scripts/judge.mjs @@ -97,8 +97,8 @@ const body = JSON.stringify({ temperature: 0, // Scales with the finding count (~250 tokens per cluster JSON entry — // member_ids + representative_id + confidence + keep + root_cause + reason — - // plus a bit of overhead). 8k tokens truncated the response at 48 findings - // (observed on minimax runs on 80bffaa72); 32k gives headroom well past 100. + // plus a bit of overhead). 32k gives headroom well past 100 findings; 8k + // truncated the response on large batches in earlier testing. max_tokens: 32000, messages: [{ role: "system", content: system }, { role: "user", content: user }], }); From 8ff7d0f3dc756bc1bcacfeb04fb4e26e035eafcd Mon Sep 17 00:00:00 2001 From: ZhenghuaBao Date: Fri, 24 Jul 2026 13:25:13 +0800 Subject: [PATCH 3/7] Address further review follow-ups on precision-filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit judge.mjs: - Reject non-numeric --threshold inputs strictly (parseFloat accepted garbage suffixes like "0.8oops" as 0.8 and hex prefixes as their integer value; use a regex + Number() combo so a typo fails loudly). - Coerce `g.confidence` via Number() before the finiteness check — a numeric-string schema drift ("0.95") would otherwise cause the finiteness test to fail and silently drop a `keep: true` group. - Resolve the representative comment by iterating [representative_id, ...member_ids] and returning the first in-range id, so a malformed group whose primary id is out of range does not skip valid members that the coverage pass has already marked as handled. postfilter.mjs: - Return to `-l` for the file-list step (single-colon output is unambiguous to parse) and do a second, file-scoped `-n` grep only for the rehome target. This dodges the ambiguity of raw `-n` output when filenames contain colons. - Treat multiple hits in the same target file as ambiguous — leave the finding on its original path instead of picking an arbitrary first line to rehome to. --- scripts/judge.mjs | 42 ++++++++++++------ scripts/postfilter.mjs | 97 +++++++++++++++++++++++++++--------------- 2 files changed, 91 insertions(+), 48 deletions(-) diff --git a/scripts/judge.mjs b/scripts/judge.mjs index ecc5323..84e1f9f 100644 --- a/scripts/judge.mjs +++ b/scripts/judge.mjs @@ -22,14 +22,20 @@ let out = null, threshold = 0.7, modelOverride = null; for (let i = 0; i < rest.length; i += 1) { if (rest[i] === "--out") out = rest[++i]; else if (rest[i] === "--threshold") { - // Validate the numeric range up front so a typo (or an unset env - // interpolation resolving to empty string) fails loudly instead of - // silently producing NaN, which — via `x >= NaN` = false — would - // drop every finding and emit an empty kept set. + // Reject anything that isn't a plain decimal in [0,1]. Using parseFloat + // would accept a trailing-garbage input ("0.8oops" → 0.8, "0x1" → 0); + // Number() is stricter but still allows leading/trailing whitespace + // and empty string, so we regex-gate the raw arg first, then coerce. + // A silent NaN or an accidentally-clamped value would otherwise drop + // every finding via `x >= NaN` = false or shift the gate entirely. const rawT = rest[++i]; - threshold = parseFloat(rawT); + if (typeof rawT !== "string" || !/^\s*-?\d+(?:\.\d+)?\s*$/.test(rawT)) { + console.error(`judge: --threshold must be a plain decimal, got ${JSON.stringify(rawT)}`); + process.exit(2); + } + threshold = Number(rawT); if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { - console.error(`judge: --threshold must be a number in [0,1], got ${JSON.stringify(rawT)}`); + console.error(`judge: --threshold must be in [0,1], got ${JSON.stringify(rawT)}`); process.exit(2); } } @@ -134,18 +140,26 @@ for (let i = 0; i < findings.length; i += 1) if (!covered.has(i)) const kept = []; const dropped = []; for (const g of groups) { - const surv = g.keep && Number.isFinite(g.confidence) && g.confidence >= threshold; - // Resolve the representative comment through both the primary id and the - // first-member fallback, but only push if we actually land on a real - // comment — a malformed judge group with an out-of-range id would - // otherwise inject `undefined` into the kept set and crash downstream. - const memberFallback = Array.isArray(g.member_ids) && g.member_ids.length ? g.member_ids[0] : null; - const repId = comments[g.representative_id] != null ? g.representative_id : memberFallback; + // Coerce confidence — the judge sometimes serializes it as a JSON string + // (e.g. "0.95") on schema drift. Without the coerce, `Number.isFinite("0.95")` + // is false and a `keep: true` group would be silently dropped. + const conf = Number(g.confidence); + const surv = g.keep && Number.isFinite(conf) && conf >= threshold; + // Resolve the representative comment by trying the primary id then every + // member_id — a malformed group whose representative_id is out of range + // must still surface a valid member, otherwise the coverage pass has + // already marked those members as "handled" and the whole group would + // vanish silently along with real findings. + const candidateIds = [g.representative_id, ...(Array.isArray(g.member_ids) ? g.member_ids : [])]; + let repId = null; + for (const id of candidateIds) { + if (Number.isInteger(id) && id >= 0 && id < comments.length && comments[id]) { repId = id; break; } + } const rep = repId != null ? comments[repId] : null; const others = Array.isArray(g.member_ids) ? g.member_ids.filter((id) => id !== repId) : []; - const line = `[conf ${g.confidence?.toFixed?.(2) ?? "?"}] ${(rep?.content || "").match(/\[(P[0-3])\]/)?.[0] || ""} ${rep?.path ?? "?"} :: ${g.root_cause} ${others.length ? "(merged " + others.length + ")" : ""}`; + const line = `[conf ${Number.isFinite(conf) ? conf.toFixed(2) : "?"}] ${(rep?.content || "").match(/\[(P[0-3])\]/)?.[0] || ""} ${rep?.path ?? "?"} :: ${g.root_cause} ${others.length ? "(merged " + others.length + ")" : ""}`; if (surv) { if (!rep) { console.error("skip malformed judge group (no valid rep): " + JSON.stringify(g).slice(0, 120)); continue; } kept.push(rep); diff --git a/scripts/postfilter.mjs b/scripts/postfilter.mjs index b3bbaee..36aba55 100644 --- a/scripts/postfilter.mjs +++ b/scripts/postfilter.mjs @@ -32,35 +32,47 @@ if (!file || !repo || !commit) { // so only clearly-non-code paths incur the drop. const KNOWN_NON_CODE = /\.(json|ya?ml|md|txt|lock|log|csv|tsv|po|pot|properties)$/i; -// Returns an array of { file, line } — each match of `pattern` at the given -// commit, with the line NUMBER as reported by git-grep -n so a rehome can -// point at the actual snippet's line rather than reusing the reviewer's -// (now-wrong) original line. +// Returns a de-duped list of files containing `pattern` at the reviewed +// commit. Uses `-l` (file-list only) so we sidestep the colon-in-filename +// ambiguity of `-n`'s ":::" format — with `-l` +// the output is just ":" and stripping the commit prefix is +// unambiguous. function grepFiles(pattern) { if (!pattern || pattern.length < 12) return []; try { - const o = execFileSync("git", ["-C", repo, "grep", "-F", "-I", "-n", "-e", pattern, commit], { + const o = execFileSync("git", ["-C", repo, "grep", "-F", "-I", "-l", "-e", pattern, commit], { encoding: "utf8", maxBuffer: 1 << 24, }); - // Output shape: ":::". Strip the commit prefix - // once (present because we asked for a specific commit), then parse. - const seen = new Map(); // file -> first line seen (dedup multi-hit files) + return [...new Set(o.split("\n").filter(Boolean).map((l) => l.slice(l.indexOf(":") + 1)))]; + } catch { return []; } +} + +// Returns line numbers where `pattern` matches inside a specific `file` at +// the commit. We pass `file` as an explicit git pathspec after `--`, so +// git-grep's output has an exact `::` prefix we can strip +// unambiguously — the colon-in-filename concern from raw `-n` output does +// not apply here because we already know the file. +function grepLinesIn(pattern, file) { + if (!pattern || pattern.length < 12) return []; + try { + const o = execFileSync( + "git", + ["-C", repo, "grep", "-F", "-I", "-n", "-e", pattern, commit, "--", file], + { encoding: "utf8", maxBuffer: 1 << 24 }, + ); + const prefix = `${commit}:${file}:`; + const lines = []; for (const raw of o.split("\n")) { - if (!raw) continue; - const afterCommit = raw.slice(raw.indexOf(":") + 1); // "::" - const firstColon = afterCommit.indexOf(":"); - if (firstColon < 0) continue; - const file = afterCommit.slice(0, firstColon); - const rest = afterCommit.slice(firstColon + 1); - const secondColon = rest.indexOf(":"); - const line = secondColon >= 0 ? Number(rest.slice(0, secondColon)) : null; - if (!seen.has(file)) seen.set(file, Number.isFinite(line) ? line : null); + if (!raw || !raw.startsWith(prefix)) continue; + const rest = raw.slice(prefix.length); + const colon = rest.indexOf(":"); + if (colon < 0) continue; + const n = Number(rest.slice(0, colon)); + if (Number.isFinite(n)) lines.push(n); } - return [...seen.entries()].map(([file, line]) => ({ file, line })); - } catch { - return []; // grep exit 1 = no match - } + return lines; + } catch { return []; } } function candidateLines(code) { @@ -78,28 +90,45 @@ const report = []; const kept = []; for (const c of comments) { - let hits = []; // [{file, line}] + // Try each candidate line from `existing_code` until one has hits. + // Remember which candidate matched so we can look up its line numbers + // in the rehome target with a second targeted grep. + let files = []; + let matchedPattern = null; for (const ln of candidateLines(c.existing_code)) { const f = grepFiles(ln); - if (f.length) { hits = f; break; } + if (f.length) { files = f; matchedPattern = ln; break; } } let path = c.path; let start_line = c.start_line; let end_line = c.end_line; let action = "keep"; - const files = hits.map((h) => h.file); - if (hits.length) { + if (files.length) { if (files.includes(c.path)) action = "keep (correct)"; - else if (hits.length === 1) { - // REHOME: the finding is really about `hits[0].file`, not the claimed - // path — update BOTH path and lines so the posted comment lands on - // the actual snippet, not the reviewer's original (unrelated) line. - const hit = hits[0]; - path = hit.file; - if (Number.isFinite(hit.line)) { start_line = hit.line; end_line = hit.line; } - action = `REHOME ${c.path} -> ${path}${Number.isFinite(hit.line) ? ":" + hit.line : ""}`; + else if (files.length === 1) { + // Single file elsewhere. Fetch the actual line NUMBERS in that file + // to update `start_line`/`end_line` on rehome — but only if the hit + // is unambiguous (exactly one line). Multiple hits in the same file + // mean the snippet appears more than once, so we cannot pick a + // single line to post on — treat as ambiguous and leave the finding + // on its original path. + const target = files[0]; + const targetLines = matchedPattern ? grepLinesIn(matchedPattern, target) : []; + if (targetLines.length === 1) { + path = target; + start_line = targetLines[0]; + end_line = targetLines[0]; + action = `REHOME ${c.path} -> ${path}:${targetLines[0]}`; + } else if (targetLines.length > 1) { + action = `keep (ambiguous: ${targetLines.length} hits in ${target})`; + } else { + // Line lookup returned nothing (rare — `-l` said match exists). + // Fall back to a path-only rehome without changing the line. + path = target; + action = `REHOME ${c.path} -> ${path} (line unresolved)`; + } } - else action = `keep (ambiguous: ${hits.length} files)`; + else action = `keep (ambiguous: ${files.length} files)`; } else if (KNOWN_NON_CODE.test(c.path)) { action = "DROP (code snippet, filed on known-non-code file, not found)"; } else { From 6006f329f9c0ee7373a2f2856dee395bdaffac93 Mon Sep 17 00:00:00 2001 From: ZhenghuaBao Date: Fri, 24 Jul 2026 14:01:05 +0800 Subject: [PATCH 4/7] Address further review follow-ups: rehome path on ambiguous line, preserve engine policy-block, clamp confidence postfilter.mjs: - On a single-file rehome where the snippet appears more than once in the target, rehome the path anyway (with the line cleared) instead of leaving the finding on the known-wrong original path. L1 has proven the code lives in the target file; the ambiguous line only means we cannot pick a single specific position. action.yml: - Snapshot $POLICY_BLOCK's existence before the L2 judge call and only clear it on L2 soft-fail if the file did not exist beforehand. Otherwise a legitimate engine-authored guardrail block would be erased by the L2 cleanup, silently turning a real block into a pass. judge.mjs: - Clamp the coerced confidence to [0,1] so an out-of-range value (e.g. `2` or a coerced boolean) cannot bypass an arbitrary threshold. - Tighten the threshold-validation comment to describe only the chosen implementation, not the alternative that was rejected. --- action.yml | 18 ++++++++++++------ scripts/judge.mjs | 16 ++++++++-------- scripts/postfilter.mjs | 10 +++++++++- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/action.yml b/action.yml index f6b7713..e1f5478 100644 --- a/action.yml +++ b/action.yml @@ -821,6 +821,12 @@ runs: echo "::warning::L1 postfilter failed — keeping engine findings" fi L2_OUT="${1%.json}.l2.json" + # Snapshot whether the engine had already written a policy-block + # BEFORE the L2 sidecar call. If the judge then soft-fails through + # its own guardrail hit, we must clear ONLY the L2-authored block + # and leave any prior engine-authored block intact — the fatal + # "Surface guardrail block" branch downstream relies on it. + if [ -f "$POLICY_BLOCK" ]; then _pb_pre_l2=1; else _pb_pre_l2=0; fi if node "$JUDGE" "$1" --model "$JUDGE_MODEL" --threshold "$JUDGE_THRESHOLD" --out "$L2_OUT" 2>&1 \ && [ -s "$L2_OUT" ]; then L2_IN_COUNT=$(node -pe "(require('$1').comments||[]).length") @@ -829,13 +835,13 @@ runs: mv "$L2_OUT" "$1" else echo "::warning::L2 judge failed — keeping L1 findings" - # If a guardrail/firewall blocked the judge request the proxy - # will have written $POLICY_BLOCK — but that block belongs to - # the L2 sidecar call, NOT to the engine's own review. Drop - # the file so the downstream "Surface guardrail block" step - # does not surface a spurious block from a soft-failed judge. - rm -f "$POLICY_BLOCK" + # Only clear POLICY_BLOCK if the file did NOT exist before the + # judge call — otherwise we would erase the engine's own + # guardrail record and turn a legitimate block into a silent + # pass on the primary review. + if [ "$_pb_pre_l2" = "0" ]; then rm -f "$POLICY_BLOCK"; fi fi + unset _pb_pre_l2 echo "::endgroup::" fi node "$CHECK" "$1" "$rc" diff --git a/scripts/judge.mjs b/scripts/judge.mjs index 84e1f9f..35d10b8 100644 --- a/scripts/judge.mjs +++ b/scripts/judge.mjs @@ -22,12 +22,9 @@ let out = null, threshold = 0.7, modelOverride = null; for (let i = 0; i < rest.length; i += 1) { if (rest[i] === "--out") out = rest[++i]; else if (rest[i] === "--threshold") { - // Reject anything that isn't a plain decimal in [0,1]. Using parseFloat - // would accept a trailing-garbage input ("0.8oops" → 0.8, "0x1" → 0); - // Number() is stricter but still allows leading/trailing whitespace - // and empty string, so we regex-gate the raw arg first, then coerce. - // A silent NaN or an accidentally-clamped value would otherwise drop - // every finding via `x >= NaN` = false or shift the gate entirely. + // Regex-gate the raw arg to require a plain decimal, then coerce via + // Number(). A silent NaN or accidentally-clamped value would otherwise + // drop every finding via `x >= NaN` = false or shift the gate entirely. const rawT = rest[++i]; if (typeof rawT !== "string" || !/^\s*-?\d+(?:\.\d+)?\s*$/.test(rawT)) { console.error(`judge: --threshold must be a plain decimal, got ${JSON.stringify(rawT)}`); @@ -142,8 +139,11 @@ const dropped = []; for (const g of groups) { // Coerce confidence — the judge sometimes serializes it as a JSON string // (e.g. "0.95") on schema drift. Without the coerce, `Number.isFinite("0.95")` - // is false and a `keep: true` group would be silently dropped. - const conf = Number(g.confidence); + // is false and a `keep: true` group would be silently dropped. Clamp + // out-of-range values to [0,1] so a stray `2` or coerced boolean cannot + // bypass any threshold. + const rawConf = Number(g.confidence); + const conf = Number.isFinite(rawConf) ? Math.min(Math.max(rawConf, 0), 1) : NaN; const surv = g.keep && Number.isFinite(conf) && conf >= threshold; // Resolve the representative comment by trying the primary id then every // member_id — a malformed group whose representative_id is out of range diff --git a/scripts/postfilter.mjs b/scripts/postfilter.mjs index 36aba55..ba2ff74 100644 --- a/scripts/postfilter.mjs +++ b/scripts/postfilter.mjs @@ -120,7 +120,15 @@ for (const c of comments) { end_line = targetLines[0]; action = `REHOME ${c.path} -> ${path}:${targetLines[0]}`; } else if (targetLines.length > 1) { - action = `keep (ambiguous: ${targetLines.length} hits in ${target})`; + // The snippet appears more than once in the target file so we cannot + // pick a single line to post on — but L1 has still proven the code + // lives in `target`, NOT in `c.path`. Rehome the path (dropping the + // now-inapplicable line) rather than leaving the finding on a + // known-wrong file just because the line is ambiguous. + path = target; + start_line = null; + end_line = null; + action = `REHOME ${c.path} -> ${path} (line ambiguous: ${targetLines.length} hits)`; } else { // Line lookup returned nothing (rare — `-l` said match exists). // Fall back to a path-only rehome without changing the line. From 310206f6e89e4b7d4bf4a232cc5b03dcf7f3845c Mon Sep 17 00:00:00 2001 From: ZhenghuaBao Date: Fri, 24 Jul 2026 16:25:32 +0800 Subject: [PATCH 5/7] Address further review follow-ups on precision-filter scripts/judge.mjs: - Extract parseStrictDecimal helper for --threshold parsing so a mistyped arg with a numeric prefix ("0.8oops", "0x1") is rejected rather than silently accepted the way parseFloat would. - Type-guard the group's confidence before Number() coercion so `true` / `[1]` / non-numeric JSON cannot slip past the clamp as `1` and bypass any threshold. Rename `conf` -> `boundedConf` at the compare site to make the clamped invariant obvious. scripts/postfilter.mjs: - Expand the known-non-code drop list to cover binary/generated assets (source maps, images, fonts, archives, native binaries, wasm, media) so a hallucinated code snippet filed on those paths is dropped rather than kept as unverified. - On the "single target file, line unresolved" rehome branch, clear the now-stale start_line / end_line. The inherited line came from the wrong file and would post on an unrelated line in the new file, or trip GitHub's inline-comment range validation. action.yml: - Snapshot POLICY_BLOCK content (not just presence) before the L2 judge call. Fact-proxy's recordPolicyBlock() can OVERWRITE the file if the judge itself hits a guardrail, which the previous presence-only flag could not detect. Restore the engine's snapshotted content on both success and failure so the engine's authoritative guardrail record wins over an L2 sidecar overwrite. Co-Authored-By: Claude Opus 4.7 (1M context) --- action.yml | 38 ++++++++++++++++++++++++----------- scripts/judge.mjs | 45 +++++++++++++++++++++++++++++------------- scripts/postfilter.mjs | 9 +++++++-- 3 files changed, 64 insertions(+), 28 deletions(-) diff --git a/action.yml b/action.yml index e1f5478..3fd0b44 100644 --- a/action.yml +++ b/action.yml @@ -821,27 +821,41 @@ runs: echo "::warning::L1 postfilter failed — keeping engine findings" fi L2_OUT="${1%.json}.l2.json" - # Snapshot whether the engine had already written a policy-block - # BEFORE the L2 sidecar call. If the judge then soft-fails through - # its own guardrail hit, we must clear ONLY the L2-authored block - # and leave any prior engine-authored block intact — the fatal - # "Surface guardrail block" branch downstream relies on it. - if [ -f "$POLICY_BLOCK" ]; then _pb_pre_l2=1; else _pb_pre_l2=0; fi + # Snapshot the engine's policy-block CONTENT (not just presence) + # before the L2 sidecar. The judge call goes through fact-proxy, + # whose `recordPolicyBlock()` can OVERWRITE `$POLICY_BLOCK` if the + # judge itself hits a guardrail — a presence-only flag can't tell + # "engine block preserved" from "engine block silently replaced by + # L2 block". After the L2 call, if we snapshotted engine content + # we always restore it (both success and fail paths) so the + # engine's guardrail record wins over an L2 sidecar overwrite. + POLICY_BLOCK_SNAPSHOT="${POLICY_BLOCK}.pre-l2" + rm -f "$POLICY_BLOCK_SNAPSHOT" + if [ -f "$POLICY_BLOCK" ]; then cp -f "$POLICY_BLOCK" "$POLICY_BLOCK_SNAPSHOT"; fi if node "$JUDGE" "$1" --model "$JUDGE_MODEL" --threshold "$JUDGE_THRESHOLD" --out "$L2_OUT" 2>&1 \ && [ -s "$L2_OUT" ]; then L2_IN_COUNT=$(node -pe "(require('$1').comments||[]).length") L2_OUT_COUNT=$(node -pe "(require('$L2_OUT').comments||[]).length") echo "L2 judge ($JUDGE_MODEL, thr=$JUDGE_THRESHOLD): $L2_IN_COUNT -> $L2_OUT_COUNT" mv "$L2_OUT" "$1" + # L2 succeeded. Restore engine's block if we snapshotted one — + # covers the case where L2 was warn-mode guardrail-hit and + # overwrote engine's authoritative record mid-call. + if [ -f "$POLICY_BLOCK_SNAPSHOT" ]; then + mv -f "$POLICY_BLOCK_SNAPSHOT" "$POLICY_BLOCK" + fi else echo "::warning::L2 judge failed — keeping L1 findings" - # Only clear POLICY_BLOCK if the file did NOT exist before the - # judge call — otherwise we would erase the engine's own - # guardrail record and turn a legitimate block into a silent - # pass on the primary review. - if [ "$_pb_pre_l2" = "0" ]; then rm -f "$POLICY_BLOCK"; fi + # L2 failed. Restore engine's block if we snapshotted one; else + # clear any L2-authored block so a sidecar guardrail hit doesn't + # surface as the engine's own block on the primary review. + if [ -f "$POLICY_BLOCK_SNAPSHOT" ]; then + mv -f "$POLICY_BLOCK_SNAPSHOT" "$POLICY_BLOCK" + else + rm -f "$POLICY_BLOCK" + fi fi - unset _pb_pre_l2 + unset POLICY_BLOCK_SNAPSHOT echo "::endgroup::" fi node "$CHECK" "$1" "$rc" diff --git a/scripts/judge.mjs b/scripts/judge.mjs index 35d10b8..0c177e9 100644 --- a/scripts/judge.mjs +++ b/scripts/judge.mjs @@ -17,24 +17,33 @@ import fs from "node:fs"; import os from "node:os"; +// Parses a strict plain decimal (e.g. "0.7", "-1", "0.5"). Returns the number +// or NaN. Deliberately does NOT use parseFloat — parseFloat stops at the +// first non-numeric character so "0.8oops" would silently become 0.8 and +// "0x1" would become 0. The regex gate rejects any trailing garbage before +// Number() coerces, so callers can trust the returned value is either a +// clean decimal or NaN. +function parseStrictDecimal(raw) { + if (typeof raw !== "string" || !/^\s*-?\d+(?:\.\d+)?\s*$/.test(raw)) return NaN; + return Number(raw); +} + const [file, ...rest] = process.argv.slice(2); let out = null, threshold = 0.7, modelOverride = null; for (let i = 0; i < rest.length; i += 1) { if (rest[i] === "--out") out = rest[++i]; else if (rest[i] === "--threshold") { - // Regex-gate the raw arg to require a plain decimal, then coerce via - // Number(). A silent NaN or accidentally-clamped value would otherwise - // drop every finding via `x >= NaN` = false or shift the gate entirely. const rawT = rest[++i]; - if (typeof rawT !== "string" || !/^\s*-?\d+(?:\.\d+)?\s*$/.test(rawT)) { + const parsed = parseStrictDecimal(rawT); + if (!Number.isFinite(parsed)) { console.error(`judge: --threshold must be a plain decimal, got ${JSON.stringify(rawT)}`); process.exit(2); } - threshold = Number(rawT); - if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) { + if (parsed < 0 || parsed > 1) { console.error(`judge: --threshold must be in [0,1], got ${JSON.stringify(rawT)}`); process.exit(2); } + threshold = parsed; } else if (rest[i] === "--model") modelOverride = rest[++i]; } @@ -138,13 +147,21 @@ const kept = []; const dropped = []; for (const g of groups) { // Coerce confidence — the judge sometimes serializes it as a JSON string - // (e.g. "0.95") on schema drift. Without the coerce, `Number.isFinite("0.95")` - // is false and a `keep: true` group would be silently dropped. Clamp - // out-of-range values to [0,1] so a stray `2` or coerced boolean cannot - // bypass any threshold. - const rawConf = Number(g.confidence); - const conf = Number.isFinite(rawConf) ? Math.min(Math.max(rawConf, 0), 1) : NaN; - const surv = g.keep && Number.isFinite(conf) && conf >= threshold; + // (e.g. "0.95") on schema drift. Type-guard first: `Number()` will happily + // return 1 for `true` and 0 for `false`/`null`/`[]`, and 1 for `[1]`, any of + // which would slip past ANY threshold and bypass the whole gate. Accept + // only real numbers or numeric strings; anything else becomes NaN and the + // group is dropped as if `keep: false`. `boundedConf` is then clamped so + // even a stray `2` from a valid-typed but out-of-range value cannot bypass. + const rawConfInput = g.confidence; + const rawConf = + typeof rawConfInput === "number" + ? rawConfInput + : typeof rawConfInput === "string" + ? parseStrictDecimal(rawConfInput) + : NaN; + const boundedConf = Number.isFinite(rawConf) ? Math.min(Math.max(rawConf, 0), 1) : NaN; + const surv = g.keep && Number.isFinite(boundedConf) && boundedConf >= threshold; // Resolve the representative comment by trying the primary id then every // member_id — a malformed group whose representative_id is out of range // must still surface a valid member, otherwise the coverage pass has @@ -159,7 +176,7 @@ for (const g of groups) { const others = Array.isArray(g.member_ids) ? g.member_ids.filter((id) => id !== repId) : []; - const line = `[conf ${Number.isFinite(conf) ? conf.toFixed(2) : "?"}] ${(rep?.content || "").match(/\[(P[0-3])\]/)?.[0] || ""} ${rep?.path ?? "?"} :: ${g.root_cause} ${others.length ? "(merged " + others.length + ")" : ""}`; + const line = `[conf ${Number.isFinite(boundedConf) ? boundedConf.toFixed(2) : "?"}] ${(rep?.content || "").match(/\[(P[0-3])\]/)?.[0] || ""} ${rep?.path ?? "?"} :: ${g.root_cause} ${others.length ? "(merged " + others.length + ")" : ""}`; if (surv) { if (!rep) { console.error("skip malformed judge group (no valid rep): " + JSON.stringify(g).slice(0, 120)); continue; } kept.push(rep); diff --git a/scripts/postfilter.mjs b/scripts/postfilter.mjs index ba2ff74..a737d47 100644 --- a/scripts/postfilter.mjs +++ b/scripts/postfilter.mjs @@ -30,7 +30,7 @@ if (!file || !repo || !commit) { // scripts with no extension) previously fell through the "not code-ext" // branch and got dropped as if they were locale files — reverse the polarity // so only clearly-non-code paths incur the drop. -const KNOWN_NON_CODE = /\.(json|ya?ml|md|txt|lock|log|csv|tsv|po|pot|properties)$/i; +const KNOWN_NON_CODE = /\.(json|ya?ml|md|txt|lock|log|csv|tsv|po|pot|properties|map|png|jpe?g|gif|svg|ico|webp|pdf|woff2?|ttf|eot|otf|zip|tar|gz|tgz|bin|exe|dll|so|dylib|class|jar|wasm|mp3|mp4|mov|wav)$/i; // Returns a de-duped list of files containing `pattern` at the reviewed // commit. Uses `-l` (file-list only) so we sidestep the colon-in-filename @@ -131,8 +131,13 @@ for (const c of comments) { action = `REHOME ${c.path} -> ${path} (line ambiguous: ${targetLines.length} hits)`; } else { // Line lookup returned nothing (rare — `-l` said match exists). - // Fall back to a path-only rehome without changing the line. + // Rehome the path but clear the line: the stale line came from the + // wrong file (`c.path`) and would either post the comment on an + // unrelated line in `target`, or trip GitHub's inline-comment range + // validation and reject the whole review. path = target; + start_line = null; + end_line = null; action = `REHOME ${c.path} -> ${path} (line unresolved)`; } } From c886f1076c32e09456ecf87005a6f8ad13c67ec4 Mon Sep 17 00:00:00 2001 From: ZhenghuaBao Date: Fri, 24 Jul 2026 16:31:37 +0800 Subject: [PATCH 6/7] Strictly coerce judge group's `keep` before threshold check Same schema-drift class as the recent `confidence` fix: when the judge serializes `keep` as the string `"false"` (or any non-boolean truthy value like `[]` / `{}`), a raw `g.keep && ...` treats it as true and the finding the judge intended to drop survives above the threshold. Accept only real boolean `true` or the literal string "true" (case- insensitive, trimmed). Anything else drops the group as a safe default, matching the confidence path where invalid input becomes NaN and drops. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/judge.mjs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/judge.mjs b/scripts/judge.mjs index 0c177e9..de8536a 100644 --- a/scripts/judge.mjs +++ b/scripts/judge.mjs @@ -161,7 +161,16 @@ for (const g of groups) { ? parseStrictDecimal(rawConfInput) : NaN; const boundedConf = Number.isFinite(rawConf) ? Math.min(Math.max(rawConf, 0), 1) : NaN; - const surv = g.keep && Number.isFinite(boundedConf) && boundedConf >= threshold; + // Coerce `keep` strictly — same schema-drift class as `confidence` above. + // Truthy JS treats the STRING `"false"` as true, so a raw `&& g.keep` would + // retain a group the judge intended to drop when the LLM stringifies the + // boolean. Accept only real `true` or the literal string "true" (case- + // insensitive); anything else drops the group as a safe default. + const keepInput = g.keep; + const groupKeep = + keepInput === true || + (typeof keepInput === "string" && keepInput.trim().toLowerCase() === "true"); + const surv = groupKeep && Number.isFinite(boundedConf) && boundedConf >= threshold; // Resolve the representative comment by trying the primary id then every // member_id — a malformed group whose representative_id is out of range // must still surface a valid member, otherwise the coverage pass has From 390a935466cdb47422cb52eae50c5527d1364d8f Mon Sep 17 00:00:00 2001 From: ZhenghuaBao Date: Fri, 24 Jul 2026 16:46:16 +0800 Subject: [PATCH 7/7] Reject out-of-range confidence + narrow JSON/YAML non-code drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/judge.mjs: - Change `boundedConf` from a clamp to a strict range check. Clamping an out-of-range value like `confidence: 2` to `1` silently promoted a schema-violating response to maximum confidence, which then survived every threshold. Reject values outside [0,1] as NaN, matching the invalid-type policy (schema violation -> drop). scripts/postfilter.mjs: - Split `KNOWN_NON_CODE` in two. `.md`, `.txt`, `.lock`, `.log`, media, and binaries are always non-code. `.json` / `.yml` / `.yaml` are ambiguous — `action.yml`, workflow YAMLs, `package.json`, `tsconfig`, k8s / IaC manifests are reviewable configuration and their findings must not be dropped just because a `git grep` on the snippet missed. Restrict the JSON / YAML drop to paths that also match a non-reviewable convention (lockfiles, locales / i18n / translations, generated build output). Extracted as `isKnownNonCode(path)` so the check remains a single call site. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/judge.mjs | 9 ++++++--- scripts/postfilter.mjs | 27 +++++++++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/scripts/judge.mjs b/scripts/judge.mjs index de8536a..2a5f9c2 100644 --- a/scripts/judge.mjs +++ b/scripts/judge.mjs @@ -151,8 +151,10 @@ for (const g of groups) { // return 1 for `true` and 0 for `false`/`null`/`[]`, and 1 for `[1]`, any of // which would slip past ANY threshold and bypass the whole gate. Accept // only real numbers or numeric strings; anything else becomes NaN and the - // group is dropped as if `keep: false`. `boundedConf` is then clamped so - // even a stray `2` from a valid-typed but out-of-range value cannot bypass. + // group is dropped. Then reject values outside the documented [0,1] range + // outright rather than clamping — a stray `2` is a schema violation, and + // silently clamping it to `1` (max confidence) lets an invalid response + // survive every threshold. Match the invalid-type policy: violation → drop. const rawConfInput = g.confidence; const rawConf = typeof rawConfInput === "number" @@ -160,7 +162,8 @@ for (const g of groups) { : typeof rawConfInput === "string" ? parseStrictDecimal(rawConfInput) : NaN; - const boundedConf = Number.isFinite(rawConf) ? Math.min(Math.max(rawConf, 0), 1) : NaN; + const boundedConf = + Number.isFinite(rawConf) && rawConf >= 0 && rawConf <= 1 ? rawConf : NaN; // Coerce `keep` strictly — same schema-drift class as `confidence` above. // Truthy JS treats the STRING `"false"` as true, so a raw `&& g.keep` would // retain a group the judge intended to drop when the LLM stringifies the diff --git a/scripts/postfilter.mjs b/scripts/postfilter.mjs index a737d47..eecfc38 100644 --- a/scripts/postfilter.mjs +++ b/scripts/postfilter.mjs @@ -26,11 +26,26 @@ if (!file || !repo || !commit) { // Only drop findings whose snippet did NOT match anywhere in the tree when // the claimed path is a KNOWN non-code file (locale JSON / doc markdown / -// changelogs / lockfiles). Extensionless code files (Dockerfile, Makefile, -// scripts with no extension) previously fell through the "not code-ext" -// branch and got dropped as if they were locale files — reverse the polarity -// so only clearly-non-code paths incur the drop. -const KNOWN_NON_CODE = /\.(json|ya?ml|md|txt|lock|log|csv|tsv|po|pot|properties|map|png|jpe?g|gif|svg|ico|webp|pdf|woff2?|ttf|eot|otf|zip|tar|gz|tgz|bin|exe|dll|so|dylib|class|jar|wasm|mp3|mp4|mov|wav)$/i; +// changelogs / lockfiles / images / binaries). Extensionless code files +// (Dockerfile, Makefile, scripts with no extension) previously fell through +// the "not code-ext" branch and got dropped as if they were locale files — +// reverse the polarity so only clearly-non-code paths incur the drop. +// +// Extensions that are ALWAYS non-code (docs, media, binaries, generated). +const DEFINITELY_NON_CODE_EXT = /\.(md|txt|lock|log|csv|tsv|po|pot|properties|map|png|jpe?g|gif|svg|ico|webp|pdf|woff2?|ttf|eot|otf|zip|tar|gz|tgz|bin|exe|dll|so|dylib|class|jar|wasm|mp3|mp4|mov|wav)$/i; +// JSON / YAML are ambiguous: action.yml, GitHub workflows, package.json, +// tsconfig.json, k8s / IaC manifests are all reviewable configuration. Only +// treat a .json / .yml / .yaml path as non-code when the path itself +// signals "lockfile", "locale/translation bundle", or "generated build +// output" — anything else keeps the finding for L2 to judge. +const CONVENTIONAL_NON_CODE_JSON_YAML_PATH = + /(?:(?:^|\/)(?:package-lock|pnpm-lock)\.(?:json|ya?ml))|(?:[-.]lock\.(?:json|ya?ml)$)|(?:(?:^|\/)(?:locales?|i18n|translations|messages|dist|build|generated|out|coverage)\/)/i; +function isKnownNonCode(p) { + if (!p) return false; + if (DEFINITELY_NON_CODE_EXT.test(p)) return true; + if (!/\.(?:json|ya?ml)$/i.test(p)) return false; + return CONVENTIONAL_NON_CODE_JSON_YAML_PATH.test(p); +} // Returns a de-duped list of files containing `pattern` at the reviewed // commit. Uses `-l` (file-list only) so we sidestep the colon-in-filename @@ -142,7 +157,7 @@ for (const c of comments) { } } else action = `keep (ambiguous: ${files.length} files)`; - } else if (KNOWN_NON_CODE.test(c.path)) { + } else if (isKnownNonCode(c.path)) { action = "DROP (code snippet, filed on known-non-code file, not found)"; } else { action = "keep (snippet unverified)";