From 7c6d80c27b7be017d46560c7b4cc0f7fa2714288 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 25 Aug 2026 00:59:16 +0900 Subject: [PATCH 1/4] fix(strix): sanitize the benign MODEL QUALITY WARNING startup banner before infra-failure detection Strix prints a box-drawn "MODEL QUALITY WARNING" banner at startup whenever the configured model is not on its own hardcoded list of recommended frontier models -- a static disclaimer about model choice, unrelated to the scan's actual outcome. The banner's literal "WARNING" text satisfies has_detected_infrastructure_error()'s generic Fatal/Denied/Warn/Warning substring matcher (and, when Strix persists its own console transcript as a report artifact, the equivalent has_strix_report_failure_signal() matcher too), so any clean, 0-vulnerability scan on the org's configured default model (nvidia_nim/nvidia/nemotron-3-super-120b-a12b, which is not on Strix's recommended list) is misclassified as a provider infrastructure failure and fails closed even though the scan itself succeeded (rc=0) with zero findings. Reproduced directly from a real PR run: TEPP#214's "strix" required check failed with "Strix run emitted provider infrastructure or failure-signal output; failing closed." while its own captured transcript shows a complete penetration test summary reporting "Low" risk posture and "Vulnerabilities 0". The same banner text (6 occurrences across fallback attempts) appears in fast-mlsirm PR #1237's strix job log, suggesting this contributes to the org-wide "zero PRs mergeable" pattern tracked in ContextualWisdomLab/.github#1212. Fix: strip the banner (matched structurally by its box-drawing delimiters plus the "MODEL QUALITY WARNING" marker line, so it does not depend on the exact recommended-model list or wrapped text) from $STRIX_LOG before has_detected_infrastructure_error() runs, and from report .log files inside sanitize_known_strix_report_warnings() before has_strix_report_failure_signal() runs -- mirroring the existing, narrowly-scoped sanitization already used for Strix's other known benign internal warning. This does not touch vulnerability-severity classification (reported_vulnerability_signal, has_blocking_vulnerability_reports, STRIX_FAIL_ON_MIN_SEVERITY): a real finding still fails closed exactly as before. Only this one cosmetic, always-present-on-non-frontier-models disclaimer is excluded from the generic infra-error matcher. New regression test console-model-quality-warning-banner-sanitized reproduces the exact TEPP#214 failure via the existing fake-strix-stub harness: fails with "Strix run emitted provider infrastructure or failure-signal output; failing closed." on the pre-fix gate script, passes on the post-fix version. Verified both directions locally (git stash of strix_quick_gate.sh alone reproduces the failure; restoring it passes) via STRIX_TEST_CASE_FILTER=console-model-quality-warning-banner-sanitized. --- scripts/ci/strix_quick_gate.sh | 56 +++++++++++++++++++++-- scripts/ci/test_strix_quick_gate.sh | 69 +++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 3 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 36ec3e5f8..5bb1baf9d 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -172,6 +172,19 @@ known_internal_warning = re.compile( r"|ended a turn without a lifecycle tool call \(interactive=False\)" r"); forcing tool continuation \(\d+/\d+\): " ) +# Strix prints this box-drawn banner at startup whenever the configured +# model is not on its own hardcoded "recommended frontier model" list. It +# is a static disclaimer about model choice, unrelated to this run's +# outcome, but its literal "WARNING" text otherwise satisfies the generic +# Fatal/Denied/Warn/Warning provider-failure-signal matcher below and +# turns every clean scan on a non-listed model into a false fail-closed. +model_quality_banner = re.compile( + r"╭─[^\n]*╮\n" + r"(?:[^\n]*\n)*?" + r"[^\n]*MODEL QUALITY WARNING[^\n]*\n" + r"(?:[^\n]*\n)*?" + r"╰─[^\n]*╯\n?" +) def iter_report_logs(root: Path): @@ -191,16 +204,52 @@ def iter_report_logs(root: Path): for log_path in iter_report_logs(root): try: - lines = log_path.read_text(encoding="utf-8").splitlines(keepends=True) + text = log_path.read_text(encoding="utf-8") except UnicodeDecodeError: continue + original = text + text = model_quality_banner.sub("", text) + lines = text.splitlines(keepends=True) filtered = [line for line in lines if not known_internal_warning.match(line)] - if filtered != lines: - log_path.write_text("".join(filtered), encoding="utf-8") + text = "".join(filtered) + if text != original: + log_path.write_text(text, encoding="utf-8") PY done } +# Strips the same benign MODEL QUALITY WARNING startup banner (see above) +# from the raw Strix console transcript so has_detected_infrastructure_error +# does not mistake it for a real provider/infrastructure failure signal. +sanitize_strix_console_log() { + local log_path="$1" + if [ -z "$log_path" ] || [ ! -f "$log_path" ] || [ -L "$log_path" ]; then + return 0 + fi + python3 - "$log_path" <<'PY' +from pathlib import Path +import re +import sys + +log_path = Path(sys.argv[1]) +model_quality_banner = re.compile( + r"╭─[^\n]*╮\n" + r"(?:[^\n]*\n)*?" + r"[^\n]*MODEL QUALITY WARNING[^\n]*\n" + r"(?:[^\n]*\n)*?" + r"╰─[^\n]*╯\n?" +) + +try: + text = log_path.read_text(encoding="utf-8") +except UnicodeDecodeError: + raise SystemExit(0) +sanitized = model_quality_banner.sub("", text) +if sanitized != text: + log_path.write_text(sanitized, encoding="utf-8") +PY +} + has_strix_report_failure_signal() { local report_root local report_log @@ -2790,6 +2839,7 @@ PY fi fi preserve_attempt_log "$model" "$rc" + sanitize_strix_console_log "$STRIX_LOG" sanitize_known_strix_report_warnings "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs" local report_failure_signal=0 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 945eb3fb3..c92b68d3d 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4542,6 +4542,35 @@ EOS echo "scan ok but unknown report warning remains" exit 0 ;; + console-model-quality-warning-banner-sanitized) + # Reproduces Strix's own startup banner, printed to the console + # (not a report artifact) whenever the configured model is not on + # its hardcoded "recommended frontier model" list. It must be + # sanitized out of $STRIX_LOG before has_detected_infrastructure_error + # runs, or a completely clean 0-vulnerability scan on a non-listed + # model is misclassified as a provider infrastructure failure. + cat <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ │ +│ MODEL QUALITY WARNING │ +│ │ +│ 'vertex_ai/console-model-quality-warning-banner-sanitized' is not a │ +│ recommended frontier model for Strix. │ +│ │ +│ You can continue, but weaker models may miss vulnerabilities or produce │ +│ lower-quality findings. │ +│ │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ │ +│ Penetration test completed │ +│ │ +│ Vulnerabilities 0 │ +│ │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + exit 0 + ;; bare-timeout-with-provider-marker) # Emit bare "Connection timed out" alongside a provider marker so # is_timeout_error() matches the Tier 3 branch gated on @@ -6368,6 +6397,16 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; + console-model-quality-warning-banner-sanitized) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/console-model-quality-warning-banner-sanitized'" \ + "1" \ + "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ @@ -10434,6 +10473,36 @@ run_gate_case "report-known-internal-warning-sanitized" \ "" \ "1" +run_gate_case "console-model-quality-warning-banner-sanitized" \ + "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/console-model-quality-warning-banner-sanitized'" \ + "1" \ + "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + run_gate_case "report-known-internal-warning-variant-sanitized" \ "vertex_ai/report-known-internal-warning-variant-sanitized" \ "" \ From bc19404852f6908ce8b95c06ebfdac1f21929bbd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 19:59:19 -0700 Subject: [PATCH 2/4] fix(strix): preserve failures around quality banner --- scripts/ci/strix_quick_gate.sh | 22 +++--- scripts/ci/test_strix_quick_gate.sh | 114 ++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 10 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 5bb1baf9d..523c35016 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -179,11 +179,12 @@ known_internal_warning = re.compile( # Fatal/Denied/Warn/Warning provider-failure-signal matcher below and # turns every clean scan on a non-listed model into a false fail-closed. model_quality_banner = re.compile( - r"╭─[^\n]*╮\n" - r"(?:[^\n]*\n)*?" - r"[^\n]*MODEL QUALITY WARNING[^\n]*\n" - r"(?:[^\n]*\n)*?" - r"╰─[^\n]*╯\n?" + r"^╭─[^\n]*╮\n" + r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" + r"^[^\n]*MODEL QUALITY WARNING[^\n]*\n" + r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" + r"^╰─[^\n]*╯\n?", + re.MULTILINE, ) @@ -233,11 +234,12 @@ import sys log_path = Path(sys.argv[1]) model_quality_banner = re.compile( - r"╭─[^\n]*╮\n" - r"(?:[^\n]*\n)*?" - r"[^\n]*MODEL QUALITY WARNING[^\n]*\n" - r"(?:[^\n]*\n)*?" - r"╰─[^\n]*╯\n?" + r"^╭─[^\n]*╮\n" + r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" + r"^[^\n]*MODEL QUALITY WARNING[^\n]*\n" + r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" + r"^╰─[^\n]*╯\n?", + re.MULTILINE, ) try: diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index c92b68d3d..924ab2469 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4542,6 +4542,20 @@ EOS echo "scan ok but unknown report warning remains" exit 0 ;; + report-model-quality-warning-preserves-prior-failure) + mkdir -p "$STRIX_REPORTS_DIR/fake-model-quality-prior-failure" + cat >"$STRIX_REPORTS_DIR/fake-model-quality-prior-failure/strix.log" <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Provider WARNING: report evidence is incomplete │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ This cosmetic banner alone is not a provider failure. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + echo "scan returned zero findings with an incomplete provider report" + exit 0 + ;; console-model-quality-warning-banner-sanitized) # Reproduces Strix's own startup banner, printed to the console # (not a report artifact) whenever the configured model is not on @@ -4568,6 +4582,26 @@ EOS │ Vulnerabilities 0 │ │ │ ╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + exit 0 + ;; + console-model-quality-warning-preserves-prior-failure) + # A real provider failure may be emitted in an earlier Strix box. + # Sanitizing the later cosmetic model-quality banner must not consume + # that preceding box or the failure signal between box boundaries. + cat <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Provider WARNING: backend returned incomplete scan evidence │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ 'vertex_ai/console-model-quality-warning-preserves-prior-failure' is not a │ +│ recommended frontier model for Strix. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Penetration test completed │ +│ Vulnerabilities 0 │ +╰──────────────────────────────────────────────────────────────────────────────╯ EOS exit 0 ;; @@ -6397,6 +6431,16 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-known-internal-warning-sanitized" \ "" ;; + report-model-quality-warning-preserves-prior-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" + ;; console-model-quality-warning-banner-sanitized) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/console-model-quality-warning-banner-sanitized" \ @@ -6407,6 +6451,16 @@ run_filtered_gate_case_if_requested() { "vertex_ai/console-model-quality-warning-banner-sanitized" \ "" ;; + console-model-quality-warning-preserves-prior-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "" + ;; provider-fatal-success-signal | provider-warning-success-signal) run_gate_case "$STRIX_TEST_CASE_FILTER" \ "vertex_ai/$STRIX_TEST_CASE_FILTER" \ @@ -10503,6 +10557,66 @@ run_gate_case "console-model-quality-warning-banner-sanitized" \ "" \ "1" +run_gate_case "console-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + +run_gate_case "report-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + run_gate_case "report-known-internal-warning-variant-sanitized" \ "vertex_ai/report-known-internal-warning-variant-sanitized" \ "" \ From c38a61f35e3eb6c7595ea14298ccf79f52ddbc36 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 20:59:38 -0700 Subject: [PATCH 3/4] fix(strix): preserve same-box failure evidence --- scripts/ci/strix_quick_gate.sh | 44 +++++----- scripts/ci/test_strix_quick_gate.sh | 128 +++++++++++++++++++++++++--- 2 files changed, 135 insertions(+), 37 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 523c35016..d14c6cd08 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -172,20 +172,12 @@ known_internal_warning = re.compile( r"|ended a turn without a lifecycle tool call \(interactive=False\)" r"); forcing tool continuation \(\d+/\d+\): " ) -# Strix prints this box-drawn banner at startup whenever the configured -# model is not on its own hardcoded "recommended frontier model" list. It -# is a static disclaimer about model choice, unrelated to this run's -# outcome, but its literal "WARNING" text otherwise satisfies the generic -# Fatal/Denied/Warn/Warning provider-failure-signal matcher below and -# turns every clean scan on a non-listed model into a false fail-closed. -model_quality_banner = re.compile( - r"^╭─[^\n]*╮\n" - r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" - r"^[^\n]*MODEL QUALITY WARNING[^\n]*\n" - r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" - r"^╰─[^\n]*╯\n?", - re.MULTILINE, -) +# Strix prints this exact box-content heading at startup whenever the +# configured model is not on its own hardcoded "recommended frontier +# model" list. Remove only that one cosmetic line: deleting the whole box +# could hide a real provider failure emitted beside the heading. +model_quality_heading = re.compile(r"│[ \t]*MODEL QUALITY WARNING[ \t]*│") +ansi_csi = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") def iter_report_logs(root: Path): @@ -209,9 +201,13 @@ for log_path in iter_report_logs(root): except UnicodeDecodeError: continue original = text - text = model_quality_banner.sub("", text) lines = text.splitlines(keepends=True) - filtered = [line for line in lines if not known_internal_warning.match(line)] + filtered = [ + line + for line in lines + if not model_quality_heading.fullmatch(ansi_csi.sub("", line.rstrip("\r\n"))) + and not known_internal_warning.match(line) + ] text = "".join(filtered) if text != original: log_path.write_text(text, encoding="utf-8") @@ -233,20 +229,18 @@ import re import sys log_path = Path(sys.argv[1]) -model_quality_banner = re.compile( - r"^╭─[^\n]*╮\n" - r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" - r"^[^\n]*MODEL QUALITY WARNING[^\n]*\n" - r"(?:(?!^╭─|^╰─)[^\n]*\n)*?" - r"^╰─[^\n]*╯\n?", - re.MULTILINE, -) +model_quality_heading = re.compile(r"│[ \t]*MODEL QUALITY WARNING[ \t]*│") +ansi_csi = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") try: text = log_path.read_text(encoding="utf-8") except UnicodeDecodeError: raise SystemExit(0) -sanitized = model_quality_banner.sub("", text) +sanitized = "".join( + line + for line in text.splitlines(keepends=True) + if not model_quality_heading.fullmatch(ansi_csi.sub("", line.rstrip("\r\n"))) +) if sanitized != text: log_path.write_text(sanitized, encoding="utf-8") PY diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 924ab2469..9959b0955 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -4556,6 +4556,17 @@ EOS echo "scan returned zero findings with an incomplete provider report" exit 0 ;; + report-model-quality-warning-preserves-same-box-failure) + mkdir -p "$STRIX_REPORTS_DIR/fake-model-quality-same-box-failure" + cat >"$STRIX_REPORTS_DIR/fake-model-quality-same-box-failure/strix.log" <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ Provider WARNING: report evidence is incomplete │ +╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + echo "scan returned zero findings with incomplete evidence in the banner box" + exit 0 + ;; console-model-quality-warning-banner-sanitized) # Reproduces Strix's own startup banner, printed to the console # (not a report artifact) whenever the configured model is not on @@ -4568,7 +4579,7 @@ EOS │ │ │ MODEL QUALITY WARNING │ │ │ -│ 'vertex_ai/console-model-quality-warning-banner-sanitized' is not a │ +│ 'vertex_ai/cosmetic-banner-sanitized' is not a │ │ recommended frontier model for Strix. │ │ │ │ You can continue, but weaker models may miss vulnerabilities or produce │ @@ -4595,13 +4606,26 @@ EOS ╰──────────────────────────────────────────────────────────────────────────────╯ ╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ │ MODEL QUALITY WARNING │ -│ 'vertex_ai/console-model-quality-warning-preserves-prior-failure' is not a │ +│ 'vertex_ai/cosmetic-banner-prior-failure' is not a │ │ recommended frontier model for Strix. │ ╰──────────────────────────────────────────────────────────────────────────────╯ ╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ │ Penetration test completed │ │ Vulnerabilities 0 │ ╰──────────────────────────────────────────────────────────────────────────────╯ +EOS + exit 0 + ;; + console-model-quality-warning-preserves-same-box-failure) + cat <<'EOS' +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ MODEL QUALITY WARNING │ +│ Provider WARNING: backend returned incomplete scan evidence │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ STRIX ──────────────────────────────────────────────────────────────────────╮ +│ Penetration test completed │ +│ Vulnerabilities 0 │ +╰──────────────────────────────────────────────────────────────────────────────╯ EOS exit 0 ;; @@ -6441,24 +6465,44 @@ run_filtered_gate_case_if_requested() { "vertex_ai/report-model-quality-warning-preserves-prior-failure" \ "" ;; + report-model-quality-warning-preserves-same-box-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" + ;; console-model-quality-warning-banner-sanitized) run_gate_case "$STRIX_TEST_CASE_FILTER" \ - "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "vertex_ai/cosmetic-banner-sanitized" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/console-model-quality-warning-banner-sanitized'" \ + "Strix run succeeded for model 'vertex_ai/cosmetic-banner-sanitized'" \ "1" \ - "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "vertex_ai/cosmetic-banner-sanitized" \ "" ;; console-model-quality-warning-preserves-prior-failure) run_gate_case "$STRIX_TEST_CASE_FILTER" \ - "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/cosmetic-banner-prior-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/cosmetic-banner-prior-failure" \ + "" + ;; + console-model-quality-warning-preserves-same-box-failure) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ "" ;; provider-fatal-success-signal | provider-warning-success-signal) @@ -10528,12 +10572,12 @@ run_gate_case "report-known-internal-warning-sanitized" \ "1" run_gate_case "console-model-quality-warning-banner-sanitized" \ - "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "vertex_ai/cosmetic-banner-sanitized" \ "" \ "0" \ - "Strix run succeeded for model 'vertex_ai/console-model-quality-warning-banner-sanitized'" \ + "Strix run succeeded for model 'vertex_ai/cosmetic-banner-sanitized'" \ "1" \ - "vertex_ai/console-model-quality-warning-banner-sanitized" \ + "vertex_ai/cosmetic-banner-sanitized" \ "" \ "vertex_ai" \ "__DEFAULT__" \ @@ -10558,12 +10602,42 @@ run_gate_case "console-model-quality-warning-banner-sanitized" \ "1" run_gate_case "console-model-quality-warning-preserves-prior-failure" \ - "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/cosmetic-banner-prior-failure" \ "" \ "1" \ "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ "1" \ - "vertex_ai/console-model-quality-warning-preserves-prior-failure" \ + "vertex_ai/cosmetic-banner-prior-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + +run_gate_case "console-model-quality-warning-preserves-same-box-failure" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/cosmetic-banner-same-box-failure" \ "" \ "vertex_ai" \ "__DEFAULT__" \ @@ -10617,6 +10691,36 @@ run_gate_case "report-model-quality-warning-preserves-prior-failure" \ "" \ "1" +run_gate_case "report-model-quality-warning-preserves-same-box-failure" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" \ + "1" \ + "Strix report artifacts emitted warning/fatal/denied/timeout output; failing closed." \ + "1" \ + "vertex_ai/report-model-quality-warning-preserves-same-box-failure" \ + "" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "" \ + "1" + run_gate_case "report-known-internal-warning-variant-sanitized" \ "vertex_ai/report-known-internal-warning-variant-sanitized" \ "" \ From 095422b40e9abd22b985bf94aa0a57ae85b6e72f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 24 Aug 2026 21:27:20 -0700 Subject: [PATCH 4/4] fix(strix): preserve raw last-attempt evidence --- scripts/ci/strix_quick_gate.sh | 28 ++++++++++++++++++---------- scripts/ci/test_strix_quick_gate.sh | 13 +++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index d14c6cd08..c8b74267f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -3,10 +3,10 @@ # automatic model fallback, transient-error retry, and severity-based # pass/fail decisions. # -# STRIX_LOG is a per-attempt temp file consumed only by -# is_transient_same_model_retry_error(); cumulative report dirs in -# STRIX_REPORTS_DIR are never overwritten. Refer to ARCHITECTURE.md -# for the 3-tier timeout classification hierarchy. +# RAW_STRIX_LOG is the immutable per-attempt console evidence. STRIX_LOG points +# to it while the scanner runs, then to a private sanitized classification copy. +# Cumulative report dirs in STRIX_REPORTS_DIR are never overwritten. Refer to +# ARCHITECTURE.md for the 3-tier timeout classification hierarchy. set -euo pipefail SCRIPT_DIR="$({ CDPATH='' && cd -P -- "$(dirname -- "$0")" && pwd -P; })" @@ -25,7 +25,9 @@ RAW_SCAN_MODE="${STRIX_SCAN_MODE:-quick}" SCAN_MODE="" ARTIFACT_REPORTS_DIR="$REPO_ROOT/strix_runs" STRIX_RUNTIME_DIR="$(mktemp -d /tmp/strix-runtime.XXXXXX)" -STRIX_LOG="$STRIX_RUNTIME_DIR/strix.log" +RAW_STRIX_LOG="$STRIX_RUNTIME_DIR/strix.log" +STRIX_CLASSIFICATION_LOG="$STRIX_RUNTIME_DIR/strix-classification.log" +STRIX_LOG="$RAW_STRIX_LOG" ACTIVE_REPORTS_DIR="$STRIX_RUNTIME_DIR/reports" ATTEMPT_LOGS_DIR="$STRIX_RUNTIME_DIR/gate-attempts" STRIX_SCAN_WORKING_DIR="$STRIX_RUNTIME_DIR/scan-cwd" @@ -128,8 +130,8 @@ publish_artifact_reports() { if [ -d "$ATTEMPT_LOGS_DIR" ] && [ ! -L "$ATTEMPT_LOGS_DIR" ]; then cp -R -- "$ATTEMPT_LOGS_DIR" "$ARTIFACT_REPORTS_DIR/gate-attempts" fi - if [ -f "$STRIX_LOG" ] && [ ! -L "$STRIX_LOG" ]; then - cp -- "$STRIX_LOG" "$ARTIFACT_REPORTS_DIR/gate-last-attempt.log" + if [ -f "$RAW_STRIX_LOG" ] && [ ! -L "$RAW_STRIX_LOG" ]; then + cp -- "$RAW_STRIX_LOG" "$ARTIFACT_REPORTS_DIR/gate-last-attempt.log" fi # Relative scanner output is copied into ACTIVE_REPORTS_DIR immediately # after each attempt and sanitized before this publication trap runs. @@ -215,8 +217,8 @@ PY done } -# Strips the same benign MODEL QUALITY WARNING startup banner (see above) -# from the raw Strix console transcript so has_detected_infrastructure_error +# Strips the same benign MODEL QUALITY WARNING startup heading (see above) +# from the private classification copy so has_detected_infrastructure_error # does not mistake it for a real provider/infrastructure failure signal. sanitize_strix_console_log() { local log_path="$1" @@ -301,7 +303,7 @@ has_strix_report_provider_failure_signal() { # shellcheck disable=SC2317,SC2329 # invoked from EXIT/INT/TERM trap cleanup_runtime() { publish_artifact_reports || true - rm -f "$STRIX_LOG" + rm -f "$RAW_STRIX_LOG" "$STRIX_CLASSIFICATION_LOG" rm -rf "$STRIX_RUNTIME_DIR" local scope_dir for scope_dir in "${PULL_REQUEST_SCOPE_DIRS[@]}"; do @@ -2557,6 +2559,10 @@ run_strix_once() { local resolved_target_path local timeout_seconds="$STRIX_PROCESS_TIMEOUT_SECONDS" local total_budget_limited_timeout=0 + # Every invocation writes an untouched raw console transcript. Classifiers + # switch to a sanitized private copy only after the raw attempt is archived. + STRIX_LOG="$RAW_STRIX_LOG" + rm -f -- "$STRIX_CLASSIFICATION_LOG" if [ "$RUN_START_EPOCH" -le 0 ]; then RUN_START_EPOCH="$(date +%s)" fi @@ -2835,6 +2841,8 @@ PY fi fi preserve_attempt_log "$model" "$rc" + cp -- "$RAW_STRIX_LOG" "$STRIX_CLASSIFICATION_LOG" + STRIX_LOG="$STRIX_CLASSIFICATION_LOG" sanitize_strix_console_log "$STRIX_LOG" sanitize_known_strix_report_warnings "$ACTIVE_REPORTS_DIR" "${resolved_target_path%/}/strix_runs" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 9959b0955..5a357794e 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -5933,6 +5933,19 @@ PY "scenario=$scenario keeps non-warning Strix report evidence" fi + if [ "$scenario" = "console-model-quality-warning-banner-sanitized" ]; then + assert_file_contains \ + "$repo_root_dir/strix_runs/gate-last-attempt.log" \ + "MODEL QUALITY WARNING" \ + "scenario=$scenario preserves the raw last-attempt console artifact" + local raw_attempt_log="" + raw_attempt_log="$(find "$repo_root_dir/strix_runs/gate-attempts" -type f -name '*.log' -print -quit 2>/dev/null || true)" + assert_file_contains \ + "$raw_attempt_log" \ + "MODEL QUALITY WARNING" \ + "scenario=$scenario preserves the raw per-attempt console artifact" + fi + if [ "$scenario" = "github-models-primary-ratelimit-fallback-success" ]; then assert_file_contains \ "$output_log" \