diff --git a/.github/workflows/strix-changed-path-quality-ci.yml b/.github/workflows/strix-changed-path-quality-ci.yml index 75e9b7d8e..31924910a 100644 --- a/.github/workflows/strix-changed-path-quality-ci.yml +++ b/.github/workflows/strix-changed-path-quality-ci.yml @@ -5,12 +5,16 @@ on: branches: [main] paths: - ".github/workflows/strix-changed-path-quality-ci.yml" + - ".github/workflows/strix.yml" - "CHANGELOG.md" - "docs/doctoring/strix-legal-git-paths.md" + - "docs/doctoring/strix-model-behavior-error.md" - "docs/doctoring/strix-quality-timeout-fixtures.md" - "scripts/ci/strix_quick_gate.sh" - "scripts/ci/test_strix_quick_gate.sh" - "tests/test_strix_changed_path_policy.py" + - "tests/test_strix_model_behavior_error.py" + - "tests/test_strix_nvidia_nim_not_found_fallback.py" - "tests/test_strix_workflow_dependency_hashes.py" - "tests/test_strix_quality_timeout_fixture_budget.py" @@ -66,6 +70,6 @@ jobs: test "$(git rev-parse HEAD)" = "${{ github.event.pull_request.head.sha || github.sha }}" python -m coverage run -m pytest tests -q bash scripts/ci/test_strix_quick_gate.sh - python -m compileall -q tests/test_strix_changed_path_policy.py tests/test_strix_workflow_dependency_hashes.py tests/test_strix_quality_timeout_fixture_budget.py + python -m compileall -q tests/test_strix_changed_path_policy.py tests/test_strix_model_behavior_error.py tests/test_strix_nvidia_nim_not_found_fallback.py tests/test_strix_workflow_dependency_hashes.py tests/test_strix_quality_timeout_fixture_budget.py bash -n scripts/ci/strix_quick_gate.sh git diff --exit-code diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 514fd8a44..b3248d943 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -853,10 +853,11 @@ jobs: # Capture the gate exit code plus its console output. The gate returns # exit 1 both for genuine blocking vulnerabilities AND for # LLM-backend-unavailable outcomes (GitHub Models "Too many requests" - # rate limits, OpenAI quota starvation, 413 tokens_limit_reached - # token-cap, connection/warm-up failures) that could not complete a scan. A backend outage is CI - # infrastructure noise, not a security finding, so it must not fail - # the required check and block merges. + # rate limits, OpenAI quota starvation, 413 tokens_limit_reached, + # connection/warm-up failures, and scanner ModelBehaviorError) that + # could not complete a scan. Provider failure is typed infrastructure + # evidence, but remains non-passing because no authoritative complete + # vulnerability result exists. strix_run_log="$RUNNER_TEMP/strix_gate_console.log" strix_rc=0 set +e @@ -876,23 +877,18 @@ jobs: fi # Recognized signals that the LLM backend was unavailable / starved. - backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|Error code:[[:space:]]*410|github_models_retirement_brownout|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404' + backend_unavailable_signal='RateLimitError|Too many requests\. For more on scraping GitHub|exceeded your current quota|insufficient_quota|billing details|"status"[[:space:]]*:[[:space:]]*"RESOURCE_EXHAUSTED"|tokens_limit_reached|Request body too large|Max size:[[:space:]]*[0-9]+[[:space:]]+tokens|Error code:[[:space:]]*413|LLM CONNECTION FAILED|Could not establish connection to the language model|LLM warm-up failed|Configured model and fallback models were unavailable|Configured Vertex model and fallback models were unavailable|emitted provider infrastructure or failure-signal output|before provider infrastructure failure|litellm(\.exceptions)?\.NotFoundError[^[:cntrl:]]*Nvidia_nimException[^[:cntrl:]]*Error code:[[:space:]]*404|Error during penetration test: loginAsGuest failed after [0-9]+ attempts: curl exit 7: curl: \(7\) Failed to connect to 127\.0\.0\.1 port 48080' + model_behavior_error_signal='(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' # Any evidence that a vulnerability was actually reported. Its presence # forces a hard failure so real findings are NEVER downgraded. Keep the # severity branch anchored away from identifiers so environment lines # such as STRIX_FAIL_ON_MIN_SEVERITY do not look like findings. reported_vulnerability_signal='Vulnerabilities[[:space:]]+[1-9]|(^|[^A-Za-z0-9_])severity[[:space:]]*:' - # The gate may already have exempted an earlier, out-of-scope - # finding (unchanged-file evidence, or below the configured minimum - # severity) and logged "allowing pipeline continuation" before - # moving on to a later, independent model attempt. That earlier - # finding's own "Vulnerabilities N" / "severity:" text must not - # poison the backend-unavailable check for a later, unrelated - # provider outage. Scope the neutral-skip decision to the log tail - # after the LAST such continuation marker (the full log when no - # exemption occurred), so an unresolved vulnerability anywhere in - # that scope still fails closed. + # An earlier out-of-scope/below-threshold finding may already have + # been exempted by the trusted gate. Classify a later provider + # outage from the tail after the last continuation marker, but keep + # that incomplete later scan non-passing. strix_neutralization_scope_log="$strix_run_log" if grep -Fq 'allowing pipeline continuation' "$strix_run_log"; then strix_neutralization_scope_log="$RUNNER_TEMP/strix_gate_console_tail.log" @@ -900,14 +896,14 @@ jobs: "$strix_run_log" > "$strix_neutralization_scope_log" fi - # Neutral skip only when ALL hold: a backend-unavailability signal is - # present and no vulnerability was reported in the relevant scope. - # This preserves real security gating while keeping uncontrollable - # provider outages from blocking current-head merge progress. - if grep -Eiq "$backend_unavailable_signal" "$strix_neutralization_scope_log" \ + # Classify provider/backend exhaustion only when no vulnerability + # finding was emitted. Classification improves diagnosis; it never + # converts an incomplete scan into passing security evidence. + if ( grep -Eiq "$backend_unavailable_signal" "$strix_neutralization_scope_log" \ + || grep -Eq "$model_behavior_error_signal" "$strix_neutralization_scope_log" ) \ && ! grep -Eiq "$reported_vulnerability_signal" "$strix_neutralization_scope_log"; then - echo "::warning title=Strix backend unavailable::Strix could not complete because its LLM backend was unavailable (rate limit / token cap / connection or warm-up failure) before producing a vulnerability report. Treating as a neutral skip so an infrastructure outage does not block merges; genuine findings still fail the check. See the strix-reports artifact and the run log." - exit 0 + echo "::error title=STRIX_PROVIDER_UNAVAILABLE::Strix could not complete authoritative vulnerability analysis because its provider/backend was unavailable (rate limit, token cap, connection, warm-up, or model-behavior failure). See the strix-reports artifact and run log." + exit "$strix_rc" fi echo "Strix reported security findings or failed for a non-backend reason; failing the required check (gate exit ${strix_rc})." >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 438bc01b5..6b0ef8d44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,10 @@ Semantic Versioning where the repository publishes a release. ### Added +- Classify Strix `ModelBehaviorError` and provider exhaustion as typed + `STRIX_PROVIDER_UNAVAILABLE` evidence while preserving a nonzero required + check. Incomplete scans and reported vulnerabilities both fail closed. + - Added an hourly organization commercial-readiness coordinator that discovers writable repositories, honors enabled dedicated writer leases and fully paginated live writer runs, refetches exact repository/workflow/run/PR state before dispatch, rotates bounded review-repair and opt-in NVIDIA OpenCode product-development targets, fails nonzero on fleet-wide inspection or dispatch outages, retains three-day JSON receipts, and keeps the existing 15-minute merge scheduler authoritative. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. - Added a dedicated OriginWeave hourly caller that invokes the product-neutral central scheduler with the exact repository, protected `main` branch, one-dispatch budget, two-hour same-head retry floor, non-cancelling single-flight heartbeat, job-scoped OIDC, and only the established scheduler credentials. @@ -51,6 +55,36 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Publish only the sanitized cumulative Strix report tree, avoiding a later + copy of relative scanner output that could reintroduce known internal warning + text into uploaded security evidence. + +- Retry configured Strix fallback models when the primary provider records a + rate-limit or infrastructure failure only in its structured report log, and + evaluate each fallback against its newest report without letting an older + failed attempt poison a complete later report. + +- Include the exact `backend/app/*.py` package context in PR-scoped Strix + scans when a module in that package changes. The trusted resolver uses a + NUL-delimited exact-head tree listing, copies unchanged dependencies from + the trusted base, and keeps changed-file attribution and provider failures + fail-closed. +- Include the exact `contextual_orchestrator/*.py` sibling-import context under + the same NUL-delimited exact-head and fail-closed path boundary without + expanding changed-file finding attribution. +- Treat Rust source and Cargo manifests as governed Strix inputs and include + trusted Cargo, toolchain, and `deny.toml` context when a workflow change + scopes a Rust workspace. +- Run Strix with an explicit canonical scan target from a temporary working + directory outside that target, so scanner state and relative reports cannot + become self-scanned source findings; preserve those reports as gate evidence. + PR-scoped Python scans also include the PostgreSQL introspection security + helpers when that package exists in the target repository. PR scopes now live + below the gate's private runtime directory so unrelated temporary-file + cleanup cannot remove scan input during PR-head materialization. +- Classify Strix `ModelBehaviorError` with zero reported vulnerabilities as + retryable model-protocol evidence, while keeping `Vulnerabilities [1-9]` and + other severity signals fail-closed. - Derived `org-queue-sweep`'s rotation index (added in `ContextualWisdomLab/.github#1220` to stop the walk-order starvation from `ContextualWisdomLab/.github#1219`) from a persistent `ORG_SWEEP_ROTATION_COUNTER` repository variable incremented by exactly one at the start of every actual sweep execution, instead of `github.run_number` (which increments on every trigger of this workflow, not only the sweep schedule — Devin review finding on `#1220`) or a wall-clock tick alone (which can repeat an offset when this single-flight, up-to-60-minute job runs behind schedule by an exact multiple of the repository count — CodeRabbit review finding on `#1223`). Falls back to the wall-clock tick only if the persistent counter itself is unavailable, so a fairness mechanism never blocks the sweep's review-dispatch/merge work. - Retried the Strix scan up to `STRIX_TRANSIENT_RETRY_PER_MODEL` times, same model, when the log shows the upstream strix-agent Caido sandbox bootstrap timing race (`loginAsGuest failed after N attempts` / `Failed to connect to 127.0.0.1 port `; tracked upstream as usestrix/strix#1036, #1037, #1056). A slow CI runner can exceed strix-agent's fixed 10-attempt sandbox-login budget before its local intercepting proxy is reachable, even though the penetration test itself never started and no vulnerability evidence was produced or lost; the Docker image is already cached from the failed attempt, so a same-model retry is cheap and typically clears the one-off boot race. Not wired into cross-model fallback, since switching LLM models cannot change local sandbox container boot timing. - Replaced nonexistent `job.workflow_repository` / `job.workflow_sha` / `job.workflow_ref` / `job.workflow_file_path` context references (actionlint: "property ... is not defined in object type") in `pr-review-fix-scheduler.yml`'s called-workflow source verification and `exact-artifact-sbom-attestation.yml`'s trusted-verifier checkout. Both always failed closed on the missing properties (ContextualWisdomLab/.github#1212) or, for the SBOM attestation checkout, silently resolved an empty repository/ref instead of the pinned trusted source (downstream `gh attestation verify --signer-repo`/`--signer-workflow`, using the separately hardcoded `SIGNER_REPOSITORY` constant rather than any workflow_ref, still failed closed on the resulting empty signer identity). `github.workflow_ref`/`github.workflow_sha` are real, documented properties, but for a `workflow_call` target they reflect the top-level *calling* workflow, not the reusable workflow's own file — a prefix match against the reusable workflow's own path can never succeed. `exact-artifact-sbom-attestation.yml`'s checkout now uses `github.workflow_sha` (correct today: it has no callers yet); `pr-review-fix-scheduler.yml`'s identity check instead validates `github.repository`, since every current caller uses a local, same-repo `uses: ./...` where caller and callee share one commit and `github.workflow_sha` is still the right pin. Tracked follow-up for the SBOM attestation checkout once a real (potentially cross-repo) caller exists: ContextualWisdomLab/.github#1228. diff --git a/docs/doctoring/strix-model-behavior-error.md b/docs/doctoring/strix-model-behavior-error.md new file mode 100644 index 000000000..449c904f4 --- /dev/null +++ b/docs/doctoring/strix-model-behavior-error.md @@ -0,0 +1,53 @@ +# Strix ModelBehaviorError classifier + +기준일: **2026-08-21** + +## Incident + +Required Strix scans can fail closed after the agent runtime raises +`ModelBehaviorError` even when the log reports `Vulnerabilities 0`. The +exception means the selected model did not follow Strix's tool-calling +protocol. Treating that protocol failure as a security finding blocked +current-head progress on otherwise empty scans. + +## Decision + +`scripts/ci/strix_quick_gate.sh` recognizes a **module-qualified** +`ModelBehaviorError` from `agents`, `pydantic_ai`, or `strix` as retryable +model evidence. A bare source-file mention is not enough. The gate moves to +the configured fallback sequence and does not retry the same model. The outer +`.github/workflows/strix.yml` classifies the failure as typed provider evidence +only when that signal is present **and** the log contains no vulnerability +evidence, while preserving the nonzero result because the scan is incomplete. + +`Vulnerabilities[[:space:]]+[1-9]` and `severity:` markers remain blocking. +Generic warnings, timeouts, provider failures, and MEDIUM-or-higher findings +are unchanged. + +## Verification contract + +`tests/test_strix_model_behavior_error.py` executes the production classifier +and the outer workflow neutralization condition against bounded synthetic +logs. It proves: + +1. a module-qualified `agents`/`pydantic_ai`/`strix` `ModelBehaviorError` + plus `Vulnerabilities 0` is retryable and typed non-passing; +2. the same exception plus `Vulnerabilities 1` stays fail-closed; +3. lowercase application prose or a bare `ModelBehaviorError` token is not + classified as the runtime exception; +4. the identifier is wired into infrastructure detection and cross-model + fallback, never same-model retry. + +## Rollback + +If a future Strix release renames the exception, add the exact new identifier +and a matching regression. Do not remove the vulnerability fail-closed guard. + +## References (APA 7th) + +GitHub. (n.d.). *Workflow syntax for GitHub Actions*. GitHub Docs. Retrieved +August 21, 2026, from +https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax + +GitHub. (n.d.). *Using workflow run logs*. GitHub Docs. Retrieved August 21, +2026, from https://docs.github.com/en/actions/how-tos/monitor-workflows/use-workflow-run-logs diff --git a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md index 70299ebdf..a088aa7ef 100644 --- a/docs/doctoring/strix-nvidia-nim-not-found-fallback.md +++ b/docs/doctoring/strix-nvidia-nim-not-found-fallback.md @@ -30,10 +30,12 @@ combining with an unrelated application `404` to spoof infrastructure fallback. Provider-side failure also remains a fail-closed incomplete scan until a distinct fallback produces complete evidence. -The outer workflow may classify exhausted provider infrastructure as neutral only -when the run log contains no vulnerability signal. Any reported severity or -non-zero vulnerability count remains blocking. Scanner reports and attempt logs -remain available as artifacts. +Exhausted provider infrastructure remains fail-closed even when the trusted +gate has classified every observed threshold finding as outside the pull +request's changed files. That classification scopes authoritative findings; it +cannot prove that an incomplete provider-exhausted scan observed every finding. +Changed, unmapped, and changed-manifest findings also remain blocking. Scanner +reports and attempt logs remain available as artifacts. ## Verification contract @@ -48,8 +50,10 @@ Regression evidence proves that: 5. model-catalog 404s enter cross-model fallback but never same-model retry; 6. the primary and first fallback are current NVIDIA hosted models; 7. GitHub Models remain later cross-provider fallbacks; -8. vulnerability signals prevent neutral infrastructure classification; and -9. the required-workflow smoke contract pins these properties. +8. provider exhaustion remains non-passing after unchanged baseline findings; +9. changed, unmapped, and changed-manifest findings also block after provider + exhaustion; and +10. the required-workflow smoke contract pins these properties. ## Limitations diff --git a/docs/doctoring/strix-pr-head-context-boundary.md b/docs/doctoring/strix-pr-head-context-boundary.md new file mode 100644 index 000000000..762fbee97 --- /dev/null +++ b/docs/doctoring/strix-pr-head-context-boundary.md @@ -0,0 +1,57 @@ +# Strix PR-head dependency context boundary + +Status: accepted 2026-08-21 + +## Incident + +The Strix run for LineageWeave PR #192 materialized changed Python files but +not the unchanged local `backend/app` dependency package. The scanner then +reported `backend.app.post_eligibility` as missing even though that module was +present in the PR head and base repository. The same changed-file-only failure +mode affected `contextual-orchestrator` PR #801: `__main__.py` imported sibling +modules omitted from the temporary scan tree. Earlier attempts also encountered +NVIDIA NIM rate limits; those provider failures must remain visible and must not +be confused with a source finding. + +TEPP PR #154 exposed the same completeness boundary for Rust: a workflow change +scoped the CI definition without the workspace's unchanged Cargo manifests, +toolchain selection, or cargo-deny policy. + +## Decision + +When a PR changes a Python module under `backend/app` or +`contextual_orchestrator`, the trusted Strix scope resolver enumerates every +Python file under that package from the exact PR head tree. It reads the Git +tree as NUL-delimited paths and applies the same +bounded path validator used for changed files, so ambiguous or unsafe entries +fail closed. The scope builder copies changed files from that head and +unchanged context from the trusted base checkout. The changed-file list +remains the finding-attribution boundary; this does not turn a context file +into a changed finding. The scan still executes only trusted scanner code and +treats PR-head blobs as non-executable data. + +This is a product-neutral extension of the existing backend context contract; +it does not replace the repository-specific context list for other backend +layouts and does not downgrade provider or vulnerability failures. + +## Evidence and rollback + +The regression fixture creates changed modules that import unchanged siblings +in both packages, then asserts that the production scope contains the +dependencies and their trusted content. Roll back this change only with an +equivalent exact-head dependency-context contract; +removing the context or weakening the Strix gate is not an acceptable rollback. + +For a workflow-scoped root Rust workspace, the behavioral fixture also requires +trusted `Cargo.toml`, `Cargo.lock`, `rust-toolchain.toml`, and `deny.toml` +contents in the materialized target. Rust source and Cargo manifests remain +governed changed inputs rather than context-only exemptions. + +## References + +National Institute of Standards and Technology. (2008). *Technical guide to +information security testing and assessment* (Special Publication 800-115). +https://doi.org/10.6028/NIST.SP.800-115 + +OWASP Foundation. (n.d.). *Web security testing guide*. Retrieved August 21, +2026, from https://owasp.org/www-project-web-security-testing-guide/ diff --git a/docs/doctoring/strix-scan-working-boundary.md b/docs/doctoring/strix-scan-working-boundary.md new file mode 100644 index 000000000..f73644c56 --- /dev/null +++ b/docs/doctoring/strix-scan-working-boundary.md @@ -0,0 +1,56 @@ +# Strix scan working-directory boundary + +## Problem + +The organization Strix gate bounded pull-request scans to a temporary scope, +but launched Strix with that scope as its current working directory. Strix +could therefore create `strix_runs/` and state files inside the tree it was +scanning. A self-generated state file was reported as a critical hard-coded +credential in a current-head `pg-erd-cloud` scan, while another scan reported a +missing unchanged DSN guard because the bounded scope omitted an imported +security helper. + +## Decision + +The gate now passes the canonical target directory as Strix's absolute `-t` +argument and runs the process from a fresh runner-temporary directory outside +the target. The temporary `strix_runs/` output is copied into the existing +active report directory after each attempt, so report classification and +artifact publication retain their previous evidence contract. The target is +never inferred from the working directory. + +When a changed backend Python file belongs to a repository that contains +`backend/app/pg_introspect`, the bounded scope includes the package's available +trusted base helpers, including `dsn_guard.py` and `introspect.py`. Repositories +without that package are unchanged. + +The bounded scope itself is created below the gate's private runtime directory. +The gate therefore owns the scope lifetime and an unrelated temporary-file +cleanup cannot remove scan input during PR-head blob materialization. + +## Verification and rollback + +`scripts/ci/test_strix_quick_gate.sh` verifies both the absolute target and the +outside working directory. It also verifies that a PostgreSQL DSN guard is +available to a scoped introspection scan. Run the shell syntax check and the +Strix quick-gate harness before publishing a central workflow change. Rollback +is a normal revert of the central PR; do not suppress changed-file attribution +or ignore scanner output to make a check green. + +The fix addresses the trust boundary between untrusted scan input and scanner +output. It does not replace exact-head review, vulnerability remediation, or +the required security workflow. + +## References + +National Institute of Standards and Technology. (2022). *Secure software +development framework (SSDF) version 1.1: Recommendations for mitigating the +risk of software vulnerabilities* (NIST Special Publication 800-218). +https://doi.org/10.6028/NIST.SP.800-218 + +MITRE. (n.d.). *CWE-22: Improper limitation of a pathname to a restricted +directory ('Path traversal')*. Common Weakness Enumeration. +https://cwe.mitre.org/data/definitions/22.html + +MITRE. (n.d.). *CWE-367: Time-of-check time-of-use (TOCTOU) race condition*. +Common Weakness Enumeration. https://cwe.mitre.org/data/definitions/367.html diff --git a/organization_commercial_readiness_fixtures.py b/organization_commercial_readiness_fixtures.py index 4275ea3dc..9d28fc592 100644 --- a/organization_commercial_readiness_fixtures.py +++ b/organization_commercial_readiness_fixtures.py @@ -90,7 +90,7 @@ def __init__( repositories: list[dict[str, Any]], snapshots: dict[str, list[RepositorySnapshot | Exception]], ) -> None: - """Initialize deterministic repository and dispatch fixtures.""" + """Initialize deterministic repository, snapshot, and dispatch fixtures.""" self.repositories = repositories self.snapshots = snapshots self.dispatched_repairs: list[tuple[str, str]] = [] diff --git a/scripts/ci/organization_commercial_readiness_loop.py b/scripts/ci/organization_commercial_readiness_loop.py index a4d7fa983..9657bd2d4 100644 --- a/scripts/ci/organization_commercial_readiness_loop.py +++ b/scripts/ci/organization_commercial_readiness_loop.py @@ -47,6 +47,9 @@ MAX_WORKFLOW_SOURCES_PER_REPOSITORY = 100 MAX_WORKFLOW_SOURCE_BYTES_PER_FILE = 1_048_576 MAX_WORKFLOW_SOURCE_BYTES_PER_REPOSITORY = 10 * 1_048_576 +SAFE_DIAGNOSTIC_METHODS = frozenset( + {"DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"} +) class GitHubError(RuntimeError): @@ -239,7 +242,7 @@ class GitHubClient: """Use the GitHub CLI as an authenticated, bounded REST transport.""" def __init__(self, token: str, *, timeout_seconds: int = 60) -> None: - """Initialize the client with one bounded GitHub credential.""" + """Initialize one authenticated GitHub credential with a bounded timeout.""" if not token: raise GitHubError("GH_TOKEN is required for organization coordination") self._token = token @@ -267,6 +270,11 @@ def request( ) -> Any: """Call one GitHub REST endpoint and decode a bounded JSON response.""" normalized_method = method.upper() + safe_method = ( + normalized_method + if normalized_method in SAFE_DIAGNOSTIC_METHODS + else "[REDACTED_METHOD]" + ) safe_path = self._redact_credential(path) args = ["gh", "api"] if normalized_method != "GET": @@ -292,7 +300,7 @@ def request( raw = (completed.stderr or completed.stdout or "GitHub API request failed").strip() bounded = self._redact_credential(raw)[-900:] raise GitHubError( - f"GitHub API {normalized_method} {safe_path} failed: {bounded}" + f"GitHub API {safe_method} {safe_path} failed: {bounded}" ) text = completed.stdout.strip() if not text: diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index ed434ff91..337373001 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -28,6 +28,8 @@ STRIX_RUNTIME_DIR="$(mktemp -d /tmp/strix-runtime.XXXXXX)" STRIX_LOG="$STRIX_RUNTIME_DIR/strix.log" ACTIVE_REPORTS_DIR="$STRIX_RUNTIME_DIR/reports" ATTEMPT_LOGS_DIR="$STRIX_RUNTIME_DIR/gate-attempts" +STRIX_SCAN_WORKING_DIR="$STRIX_RUNTIME_DIR/scan-cwd" +STRIX_SCAN_OUTPUT_DIR="$STRIX_SCAN_WORKING_DIR/strix_runs" STRIX_REPORTS_DIR="$ACTIVE_REPORTS_DIR" STRIX_PROCESS_TIMEOUT_SECONDS="${STRIX_PROCESS_TIMEOUT_SECONDS:-1200}" STRIX_TOTAL_TIMEOUT_SECONDS="${STRIX_TOTAL_TIMEOUT_SECONDS:-0}" @@ -129,13 +131,8 @@ publish_artifact_reports() { if [ -f "$STRIX_LOG" ] && [ ! -L "$STRIX_LOG" ]; then cp -- "$STRIX_LOG" "$ARTIFACT_REPORTS_DIR/gate-last-attempt.log" fi - local scope_dir scope_reports_dir - for scope_dir in "${PULL_REQUEST_SCOPE_DIRS[@]}"; do - scope_reports_dir="$scope_dir/strix_runs" - if [ -d "$scope_reports_dir" ] && [ ! -L "$scope_reports_dir" ]; then - cp -R -- "$scope_reports_dir"/. "$ARTIFACT_REPORTS_DIR"/ - fi - done + # Relative scanner output is copied into ACTIVE_REPORTS_DIR immediately + # after each attempt and sanitized before this publication trap runs. } preserve_attempt_log() { @@ -211,6 +208,18 @@ has_strix_report_failure_signal() { if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then continue fi + # A fallback attempt must be judged by its own newest structured report. + # Older attempt directories remain published for audit evidence, but a + # provider warning from an earlier failed model must not poison a complete + # later fallback report. + if [ "$report_root" = "$STRIX_REPORTS_DIR" ]; then + local newest_report_root + newest_report_root="$(latest_strix_report_dir 2>/dev/null || true)" + if [ -z "$newest_report_root" ]; then + continue + fi + report_root="$newest_report_root" + fi while IFS= read -r -d '' report_log; do if grep -Eiq '(^|[^[:alpha:]])(Fatal|Denied|Warn|Warning|WARNING|Timeout)([^[:alpha:]]|$)' "$report_log"; then return 0 @@ -220,6 +229,30 @@ has_strix_report_failure_signal() { return 1 } +has_strix_report_provider_failure_signal() { + local report_root + local report_log + for report_root in "$@"; do + if [ -z "$report_root" ] || [ ! -d "$report_root" ] || [ -L "$report_root" ]; then + continue + fi + if [ "$report_root" = "$STRIX_REPORTS_DIR" ]; then + local newest_report_root + newest_report_root="$(latest_strix_report_dir 2>/dev/null || true)" + if [ -z "$newest_report_root" ]; then + continue + fi + report_root="$newest_report_root" + fi + while IFS= read -r -d '' report_log; do + if grep -Eiq 'RateLimitError|Nvidia_nimException|Too Many Requests|Error code:[[:space:]]*429|provider.{0,80}(unavailable|exhausted|rate.?limit|timeout|connection)' "$report_log"; then + return 0 + fi + done < <(find "$report_root" -type f -name '*.log' -print0) + done + return 1 +} + # shellcheck disable=SC2317,SC2329 # invoked from EXIT/INT/TERM trap cleanup_runtime() { publish_artifact_reports || true @@ -235,6 +268,16 @@ cleanup_runtime() { trap cleanup_runtime EXIT INT TERM +make_pull_request_scope_dir() { + local scope_parent="$STRIX_RUNTIME_DIR/pr-scopes" + if [ -L "$scope_parent" ]; then + echo "ERROR: pull request scope parent must not be a symlink." >&2 + return 2 + fi + mkdir -p -- "$scope_parent" + mktemp -d "$scope_parent/strix-pr-scope.XXXXXX" +} + STRIX_LLM_FILE="${STRIX_LLM_FILE:-}" if [ -z "$STRIX_LLM_FILE" ]; then echo "ERROR: STRIX_LLM_FILE must reference a regular file containing the model." >&2 @@ -616,7 +659,7 @@ copy_pr_head_blob_to_file() { is_supported_source_file() { case "$1" in - *.java | *.kt | *.kts | *.groovy | *.scala | *.py | *.js | *.jsx | *.ts | *.tsx | *.vue | *.yaml | *.yml | *.sh | *.sql | *.xml | *.json | *.html | *.css | *.md) + *.java | *.kt | *.kts | *.groovy | *.scala | *.rs | *.py | *.js | *.jsx | *.ts | *.tsx | *.vue | *.yaml | *.yml | *.sh | *.sql | *.xml | *.json | *.html | *.css | *.md) return 0 ;; Dockerfile | */Dockerfile | Dockerfile.* | */Dockerfile.* | Containerfile | */Containerfile | Makefile | */Makefile) @@ -630,7 +673,7 @@ is_supported_source_file() { is_dependency_manifest_path() { case "$1" in - pom.xml | */pom.xml | package.json | */package.json | package-lock.json | */package-lock.json | pnpm-lock.yaml | */pnpm-lock.yaml | yarn.lock | */yarn.lock | pyproject.toml | */pyproject.toml | requirements.txt | */requirements.txt | requirements-*.txt | */requirements-*.txt | uv.lock | */uv.lock) + pom.xml | */pom.xml | Cargo.toml | */Cargo.toml | Cargo.lock | */Cargo.lock | package.json | */package.json | package-lock.json | */package-lock.json | pnpm-lock.yaml | */pnpm-lock.yaml | yarn.lock | */yarn.lock | pyproject.toml | */pyproject.toml | requirements.txt | */requirements.txt | requirements-*.txt | */requirements-*.txt | uv.lock | */uv.lock) return 0 ;; *) @@ -1186,6 +1229,8 @@ is_scannable_changed_file() { pull_request_scope_context_files() { local needs_backend_python=0 + local needs_backend_app_python=0 + local needs_contextual_orchestrator_python=0 local needs_frontend_email_api_context=0 local needs_deployment_context=0 local changed_file normalized_changed_file @@ -1196,6 +1241,12 @@ pull_request_scope_context_files() { if [[ "$normalized_changed_file" =~ ^backend/.+\.py$ ]]; then needs_backend_python=1 fi + if [[ "$normalized_changed_file" =~ ^backend/app/.+\.py$ ]]; then + needs_backend_app_python=1 + fi + ;; + contextual_orchestrator/*.py) + needs_contextual_orchestrator_python=1 ;; # The app shell, email components, threading URL builder, and API client can # shape frontend email retrieval flows; include backend auth context with them. @@ -1259,6 +1310,80 @@ backend/services/llm_provider_urls.py backend/services/text_safety.py backend/services/threading_service.py EOF + # PostgreSQL introspection helpers are a security boundary for repositories + # that expose this package. Include their trusted base copies when present; + # the conditional keeps the shared gate usable by repositories without it. + local context_file + for context_file in \ + backend/app/pg_introspect/__init__.py \ + backend/app/pg_introspect/column_examples.py \ + backend/app/pg_introspect/dsn_guard.py \ + backend/app/pg_introspect/forward_ddl.py \ + backend/app/pg_introspect/introspect.py \ + backend/app/pg_introspect/queries.py \ + backend/app/pg_introspect/snapshot_collect.py; do + if [ -f "$REPO_ROOT/$context_file" ] && [ ! -L "$REPO_ROOT/$context_file" ]; then + printf '%s\n' "$context_file" + fi + done + fi + + if [ "$needs_backend_app_python" -eq 1 ]; then + local backend_app_head_sha + backend_app_head_sha="$(trim_whitespace "${PR_HEAD_SHA:-}")" + if { [ -z "$backend_app_head_sha" ] || ! is_valid_git_commit_sha "$backend_app_head_sha"; } && pull_request_head_blob_required; then + echo "ERROR: backend/app PR-head context requires an exact head SHA; failing closed." >&2 + return 2 + elif [ -n "$backend_app_head_sha" ] && is_valid_git_commit_sha "$backend_app_head_sha"; then + local backend_app_tree_file context_file normalized_context_file + backend_app_tree_file="$(mktemp "${RUNNER_TEMP:-/tmp}/strix-backend-app-context.XXXXXX")" || return 2 + if ! git -c core.quotepath=false ls-tree -rz --name-only "$backend_app_head_sha" -- backend/app >"$backend_app_tree_file"; then + rm -f -- "$backend_app_tree_file" + echo "ERROR: backend/app PR-head context could not be enumerated; failing closed." >&2 + return 2 + fi + while IFS= read -r -d '' context_file; do + normalized_context_file="$(normalize_changed_file_path "$context_file")" || { + rm -f -- "$backend_app_tree_file" + return 2 + } + case "$normalized_context_file" in + backend/app/*.py) + printf '%s\n' "$normalized_context_file" + ;; + esac + done <"$backend_app_tree_file" + rm -f -- "$backend_app_tree_file" + fi + fi + + if [ "$needs_contextual_orchestrator_python" -eq 1 ]; then + local contextual_orchestrator_head_sha + contextual_orchestrator_head_sha="$(trim_whitespace "${PR_HEAD_SHA:-}")" + if { [ -z "$contextual_orchestrator_head_sha" ] || ! is_valid_git_commit_sha "$contextual_orchestrator_head_sha"; } && pull_request_head_blob_required; then + echo "ERROR: contextual_orchestrator PR-head context requires an exact head SHA; failing closed." >&2 + return 2 + elif [ -n "$contextual_orchestrator_head_sha" ] && is_valid_git_commit_sha "$contextual_orchestrator_head_sha"; then + local contextual_orchestrator_tree_file context_file normalized_context_file + contextual_orchestrator_tree_file="$(mktemp "${RUNNER_TEMP:-/tmp}/strix-contextual-orchestrator-context.XXXXXX")" || return 2 + if ! git -c core.quotepath=false ls-tree -rz --name-only "$contextual_orchestrator_head_sha" -- contextual_orchestrator >"$contextual_orchestrator_tree_file"; then + rm -f -- "$contextual_orchestrator_tree_file" + echo "ERROR: contextual_orchestrator PR-head context could not be enumerated; failing closed." >&2 + return 2 + fi + while IFS= read -r -d '' context_file; do + normalized_context_file="$(normalize_changed_file_path "$context_file")" || { + rm -f -- "$contextual_orchestrator_tree_file" + return 2 + } + case "$normalized_context_file" in + contextual_orchestrator/*.py) + printf '%s\n' "$normalized_context_file" + ;; + esac + done <"$contextual_orchestrator_tree_file" + rm -f -- "$contextual_orchestrator_tree_file" + fi fi if [ "$needs_frontend_email_api_context" -eq 1 ]; then @@ -1290,6 +1415,17 @@ docker-compose.yml render.yaml VERSION EOF + # Workflow changes in a Rust workspace need dependency, toolchain, and + # policy context so Strix can analyze the repository as a complete unit. + if [ -f "$REPO_ROOT/Cargo.toml" ]; then + cat <<'EOF' +Cargo.toml +Cargo.lock +rust-toolchain.toml +rust-toolchain +deny.toml +EOF + fi fi } @@ -1306,7 +1442,7 @@ changed_file_list_contains() { build_pull_request_scope_dir() { local scope_dir - scope_dir="$(mktemp -d "${TMPDIR:-/tmp}/strix-pr-scope.XXXXXX")" + scope_dir="$(make_pull_request_scope_dir)" || return 2 scope_dir="$({ CDPATH='' && cd -P -- "$scope_dir" && pwd -P; })" PULL_REQUEST_SCOPE_DIRS+=("$scope_dir") @@ -1479,7 +1615,7 @@ PY build_pull_request_head_tree_scope_dir() { local scope_dir - scope_dir="$(mktemp -d "${TMPDIR:-/tmp}/strix-pr-scope.XXXXXX")" + scope_dir="$(make_pull_request_scope_dir)" || return 2 scope_dir="$({ CDPATH='' && cd -P -- "$scope_dir" && pwd -P; })" PULL_REQUEST_SCOPE_DIRS+=("$scope_dir") @@ -2379,7 +2515,7 @@ run_strix_once() { STRIX_CHILD_EXECUTABLE_ROOT="$STRIX_EXECUTABLE_ROOT" \ STRIX_CHILD_EXECUTABLE_SHA256="$STRIX_EXECUTABLE_SHA256" \ STRIX_CHILD_REQUIRE_EXECUTABLE_INTEGRITY="${IS_PR_EVIDENCE_RUN:-false}" \ - python3 - "$timeout_seconds" "$resolved_target_path" "$SCAN_MODE" "$STRIX_LOG" <<'PY' +python3 - "$timeout_seconds" "$resolved_target_path" "$SCAN_MODE" "$STRIX_LOG" "$STRIX_SCAN_WORKING_DIR" <<'PY' import hashlib import hmac import os @@ -2393,6 +2529,7 @@ timeout_seconds = int(sys.argv[1]) target_path = sys.argv[2] scan_mode = sys.argv[3] log_path = pathlib.Path(sys.argv[4]) +scan_working_dir = pathlib.Path(sys.argv[5]) # Failure classifiers read this path even when trusted executable or target # validation fails before a child process starts. Materialize it first so the # primary log shows one configuration error instead of repeated grep noise. @@ -2532,12 +2669,29 @@ if any(ch in str(target_cwd) for ch in ("\x00", "\n", "\r")): sys.stderr.write("ERROR: Strix target path contains unsupported control characters.\n") raise SystemExit(2) -command = [resolved_strix_bin, "-n", "-t", ".", "--scan-mode", scan_mode] +if scan_working_dir.is_symlink(): + sys.stderr.write("ERROR: Strix scan working directory must not be a symlink.\n") + raise SystemExit(2) +scan_working_dir.mkdir(parents=True, exist_ok=True) +scan_output_dir = scan_working_dir / "strix_runs" +if scan_output_dir.is_symlink(): + sys.stderr.write("ERROR: Strix scan output directory must not be a symlink.\n") + raise SystemExit(2) +if scan_output_dir.exists(): + import shutil + + shutil.rmtree(scan_output_dir) +scan_output_dir.mkdir() + +# Keep scanner-created state and relative report files outside the untrusted +# scan target. The target remains explicit and absolute, so changing cwd cannot +# change which source tree is scanned. +command = [resolved_strix_bin, "-n", "-t", str(target_cwd), "--scan-mode", scan_mode] try: process = subprocess.Popen( command, - cwd=str(target_cwd), + cwd=str(scan_working_dir), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, @@ -2570,6 +2724,9 @@ except subprocess.TimeoutExpired: PY rc=$? set -e + if [ -d "$STRIX_SCAN_OUTPUT_DIR" ] && [ ! -L "$STRIX_SCAN_OUTPUT_DIR" ]; then + cp -R -- "$STRIX_SCAN_OUTPUT_DIR"/. "$ACTIVE_REPORTS_DIR"/ + fi local end_epoch end_epoch="$(date +%s)" local elapsed=$((end_epoch - start_epoch)) @@ -2664,6 +2821,17 @@ is_nvidia_nim_not_found_error() { return 1 } +is_model_behavior_error() { + # Classify only a module-qualified Strix/Agents SDK protocol exception. + # A bare source-file mention of ModelBehaviorError is not retryable. + # Cross-model fallback may continue; same-model retry does not. + if grep -Eq '(^|[^A-Za-z0-9_])(agents|pydantic_ai|strix)(\.[A-Za-z_][A-Za-z0-9_]*)*\.ModelBehaviorError([^A-Za-z0-9_]|$)' "$STRIX_LOG"; then + return 0 + fi + + return 1 +} + ## Determines whether the last strix failure is a transient error eligible ## for same-model retry (up to STRIX_TRANSIENT_RETRY_PER_MODEL times). ## Five error families qualify: @@ -2821,6 +2989,18 @@ strix_log_has_github_models_context() { } is_github_models_unavailable_model_error() { + # GitHub Models may retire a provider model with HTTP 410. Treat that as a + # bounded family-unavailable signal only when one physical provider-error + # line carries all three facts: an anchored LiteLLM/OpenAI exception, trusted + # GitHub Models context, and a complete HTTP 410 token. Anchoring the provider + # exception prevents target/repository output prefixes from spoofing fallback; + # the non-digit boundary rejects numeric continuations such as 4100/4104. + if grep -Ei '^[[:space:]]*(Error:[[:space:]]*)?((litellm(\.exceptions)?|openai)\.[A-Za-z0-9_]*(Error|Exception)|OpenAIException)([[:space:]:-]|$)' "$STRIX_LOG" | + grep -Ei '(models\.github\.ai|GitHub Models|github_models)' | + grep -Eq 'HTTP[[:space:]]+410([^0-9]|$)'; then + return 0 + fi + if grep -Eiq 'Unavailable model:[[:space:]]*[^[:space:]]+' "$STRIX_LOG" && grep -Eiq '(litellm\.BadRequestError|OpenAIException|LLM CONNECTION FAILED|Could not establish connection to the language model|models\.github\.ai|GitHub Models|openai)' "$STRIX_LOG"; then return 0 @@ -3003,6 +3183,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + if is_caido_bootstrap_timing_error; then return 0 fi @@ -3857,6 +4041,10 @@ is_model_retryable_error() { return 0 fi + if is_model_behavior_error; then + return 0 + fi + if is_github_models_api_compatible_model "$model" && is_github_models_unavailable_model_error; then return 0 fi @@ -3888,6 +4076,16 @@ is_model_retryable_error() { return 0 fi + # A provider failure can be recorded only in Strix's structured report log. + # run_strix_once already marks that evidence as infrastructure failure, but + # the child stdout log used by the classifiers may not contain the provider + # exception. In strict mode, let configured distinct fallbacks run instead of + # treating the report-only signal as a non-recoverable source failure. + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && provider_signal_fail_closed_enabled && + has_strix_report_provider_failure_signal "$ACTIVE_REPORTS_DIR" "${TARGET_PATH%/}/strix_runs"; then + return 0 + fi + if [ "$PR_FINDINGS_DECISION" = "retry_model_inconsistency" ]; then return 0 fi @@ -4049,7 +4247,7 @@ run_current_target_scan() { echo "Strix quick scan failed with a non-recoverable error." >&2 return 1 fi - done + done if should_fail_pull_request_infra_zero_findings; then return 1 @@ -4071,6 +4269,12 @@ run_current_target_scan() { return 1 fi + if [ "$INFRA_ERROR_DETECTED" -eq 1 ] && + [ "$PR_FINDINGS_DECISION" = "allow_baseline" ]; then + echo "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." >&2 + return 1 + fi + local threshold_rank threshold_rank="$(severity_rank "$STRIX_FAIL_ON_MIN_SEVERITY")" if [ "${STRIX_MAX_SEVERITY_RANK:--1}" -ge "$threshold_rank" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 3d622ac73..bf0a8693e 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -171,9 +171,22 @@ assert_strix_pr_scope_includes_deployment_context() { assert_file_contains "$GATE_SCRIPT" "frontend/package-lock.json" "strix gate includes frontend dependency lock context" assert_file_contains "$GATE_SCRIPT" "frontend/postcss.config.mjs" "strix gate includes frontend build config context" assert_file_contains "$GATE_SCRIPT" "VERSION" "strix gate includes release version context for workflow scans" + assert_file_contains "$GATE_SCRIPT" "*.rs" "strix gate recognizes Rust source files" + assert_file_contains "$GATE_SCRIPT" "Cargo.toml | */Cargo.toml | Cargo.lock | */Cargo.lock" "strix gate recognizes Rust dependency manifests" + assert_file_contains "$GATE_SCRIPT" 'if [ -f "$REPO_ROOT/Cargo.toml" ]; then' "strix gate detects Rust workspaces for workflow scan context" + assert_file_contains "$GATE_SCRIPT" "rust-toolchain.toml" "strix gate includes Rust toolchain context for workflow scans" + assert_file_contains "$GATE_SCRIPT" "deny.toml" "strix gate includes Rust dependency policy context for workflow scans" assert_file_contains "$GATE_SCRIPT" "scripts/ci/test_*.sh" "strix gate excludes large CI self-test harnesses from PR scan targets" } +assert_strix_pr_scope_includes_contextual_orchestrator_context() { + assert_file_contains "$GATE_SCRIPT" "needs_contextual_orchestrator_python=0" "strix gate tracks contextual-orchestrator package context" + assert_file_contains "$GATE_SCRIPT" 'contextual_orchestrator/*.py)' "strix gate detects contextual-orchestrator Python changes" + assert_file_contains "$GATE_SCRIPT" 'git -c core.quotepath=false ls-tree -rz --name-only "$contextual_orchestrator_head_sha" -- contextual_orchestrator' "strix gate enumerates contextual-orchestrator context from the exact PR head" + assert_file_contains "$GATE_SCRIPT" 'contextual_orchestrator_tree_file="$(mktemp' "strix gate bounds contextual-orchestrator context enumeration in a private file" + assert_file_contains "$GATE_SCRIPT" 'rm -f -- "$contextual_orchestrator_tree_file"' "strix gate cleans contextual-orchestrator context enumeration evidence" +} + assert_strix_workflow_pr_trigger_hardened() { local workflow_file="$REPO_ROOT/.github/workflows/strix.yml" @@ -480,9 +493,12 @@ assert_strix_llm_file_read_is_literal_data() { } assert_strix_child_target_uses_constant_argument() { - assert_file_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", ".", "--scan-mode", scan_mode]' "strix gate passes a constant target argument to the child process" - assert_file_contains "$GATE_SCRIPT" 'cwd=str(target_cwd)' "strix gate runs the child process from the canonical target directory" - assert_file_not_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", target_path, "--scan-mode", scan_mode]' "strix gate must not forward raw target paths as child arguments" + assert_file_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", str(target_cwd), "--scan-mode", scan_mode]' "strix gate passes the canonical target argument to the child process" + assert_file_contains "$GATE_SCRIPT" 'cwd=str(scan_working_dir)' "strix gate runs the child process outside the scan target" + assert_file_contains "$GATE_SCRIPT" 'make_pull_request_scope_dir()' "strix gate creates PR scopes under its private runtime directory" + assert_file_contains "$GATE_SCRIPT" 'scope_parent="$STRIX_RUNTIME_DIR/pr-scopes"' "strix gate keeps PR scopes inside the private runtime directory" + assert_file_not_contains "$GATE_SCRIPT" 'command = [resolved_strix_bin, "-n", "-t", ".", "--scan-mode", scan_mode]' "strix gate must not rely on the child cwd as its scan target" + assert_file_not_contains "$GATE_SCRIPT" 'cwd=str(target_cwd)' "strix gate must not run the child process inside the scan target" } assert_opencode_review_uses_codegraph_and_gpt5_fallback() { @@ -3304,6 +3320,18 @@ success|runtime-env-forwarding|vertex-primary-success-timing-message|direct-open echo "scan ok" exit 0 ;; + scan-working-directory-isolated) + if [ "$PWD" = "$target_path" ] || [[ "$PWD" == "$target_path"/* ]]; then + echo "Error: Strix process inherited the untrusted scan target as cwd" >&2 + exit 81 + fi + if [ ! -f "$target_path/backend/app/pg_introspect/dsn_guard.py" ]; then + echo "Error: PostgreSQL DSN guard context missing from PR scope" >&2 + exit 82 + fi + echo "scan ok with isolated Strix working directory" + exit 0 + ;; success-with-critical-report) mkdir -p "$STRIX_REPORTS_DIR/fake-success/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-success/vulnerabilities/vuln-0001.md" <<'REPORT' @@ -3723,6 +3751,44 @@ REPORT ;; esac ;; + github-models-http410-authenticated-fallback-success | github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) + case "${STRIX_LLM:-}" in + openai/gpt-5) + case "${FAKE_STRIX_SCENARIO:?}" in + github-models-http410-authenticated-fallback-success) + echo "Error: litellm.BadRequestError: GitHub Models provider error at models.github.ai/inference: HTTP 410 Gone" + ;; + github-models-http410-missing-http-token) + echo "Error: litellm.BadRequestError: GitHub Models provider retirement at models.github.ai/inference" + ;; + github-models-http410-missing-provider-error) + echo "GitHub Models response at models.github.ai/inference: HTTP 410 Gone" + ;; + github-models-http410-numeric-continuation-4100) + echo "Error: litellm.BadRequestError: GitHub Models provider error at models.github.ai/inference: HTTP 4100" + ;; + github-models-http410-numeric-continuation-4104) + echo "Error: litellm.BadRequestError: GitHub Models provider error at models.github.ai/inference: HTTP 4104" + ;; + github-models-http410-target-output-spoof) + echo "TARGET OUTPUT: Error: litellm.BadRequestError: GitHub Models provider error HTTP 410" + ;; + github-models-retirement-brownout-phrase-only) + echo "GitHub Models retirement brownout" + ;; + esac + exit 1 + ;; + openai/deepseek/deepseek-r1-0528) + echo "scan ok after authenticated GitHub Models HTTP 410 retirement" + exit 0 + ;; + *) + echo "Error: GitHub Models HTTP 410 fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 39 + ;; + esac + ;; github-models-primary-ratelimit-fallback-success) case "${STRIX_LLM:-}" in openai/gpt-5) @@ -3741,7 +3807,7 @@ REPORT ;; esac ;; - github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) + github-models-fallback-provider-signal-tries-next | github-models-fallback-baseline-vulnerability-before-next-success-continues | github-models-exhausted-after-baseline-vulnerability-fails-closed | github-models-fallback-changed-vulnerability-before-next-success-blocks | github-models-fallback-dockerfile-test-baseline-before-next-success-continues) case "${STRIX_LLM:-}" in openai/gpt-5) echo "LLM CONNECTION FAILED" @@ -3750,7 +3816,8 @@ REPORT exit 1 ;; openai/deepseek/deepseek-r1-0528) - if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-baseline-vulnerability-before-next-success-continues" ]; then + if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-fallback-baseline-vulnerability-before-next-success-continues" ] || + [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-fails-closed" ]; then mkdir -p "$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities" cat >"$STRIX_REPORTS_DIR/fake-pr-baseline-provider-signal/vulnerabilities/vuln-0001.md" <<'EOS' Severity: CRITICAL @@ -3779,6 +3846,12 @@ EOS exit 2 ;; openai/deepseek/deepseek-v3-0324) + if [ "${FAKE_STRIX_SCENARIO:?}" = "github-models-exhausted-after-baseline-vulnerability-fails-closed" ]; then + echo "LLM CONNECTION FAILED" + echo "Could not establish connection to the language model." + echo "Error: provider retirement brownout" + exit 1 + fi echo "scan ok after second GitHub Models fallback" exit 0 ;; @@ -4406,11 +4479,37 @@ EOS echo "Denied: provider credentials were rejected" exit 0 ;; + provider-report-rate-limit-fallback-success) + case "${STRIX_LLM:-}" in + vertex_ai/report-rate-limit-primary) + mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit" + cat >"$STRIX_REPORTS_DIR/fake-report-rate-limit/strix.log" <<'EOS' +2026-08-21 04:00:00.000 WARNING strix-pr-scope-example - strix.provider: RateLimitError: provider response was exhausted +EOS + echo "scan aborted after provider report-rate-limit signal" + exit 1 + ;; + vertex_ai/fallback-one) + mkdir -p "$STRIX_REPORTS_DIR/fake-report-rate-limit-fallback" + echo "scan ok after report-only provider fallback" + exit 0 + ;; + *) + echo "Error: report-only provider fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 60 + ;; + esac + ;; report-known-internal-warning-sanitized) mkdir -p "$STRIX_REPORTS_DIR/fake-known-internal-warning" cat >"$STRIX_REPORTS_DIR/fake-known-internal-warning/strix.log" <<'EOS' 2026-06-18 13:08:05.986 WARNING strix-pr-scope-example - strix.core.execution: agent a9fb4033 produced non-lifecycle final output in non-interactive mode; forcing tool continuation (1/500): internal agent coordination note 2026-06-18 13:10:44.089 INFO strix-pr-scope-example - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s) +EOS + mkdir -p strix_runs/fake-known-internal-warning-relative + cat >strix_runs/fake-known-internal-warning-relative/strix.log <<'EOS' +2026-06-18 13:08:05.986 WARNING strix-pr-scope-example - strix.core.execution: agent a9fb4033 produced non-lifecycle final output in non-interactive mode; forcing tool continuation (1/500): relative internal agent coordination note +2026-06-18 13:10:44.089 INFO strix-pr-scope-example - strix.tools.finish.tool: finish_scan: completed scan with 0 vulnerability report(s) EOS outside_report_dir="${FAKE_STRIX_OUTSIDE_REPORT_DIR:-$(dirname -- "$STRIX_REPORTS_DIR")/outside-strix-report}" mkdir -p "$outside_report_dir" @@ -5125,6 +5224,20 @@ EOS echo "scan ok with deployment entrypoint context" exit 0 ;; + pr-rust-workspace-context) + for rust_context in Cargo.toml Cargo.lock rust-toolchain.toml deny.toml; do + if [ ! -f "$target_path/$rust_context" ]; then + echo "Error: Rust workflow scope missing $rust_context ($target_path)" >&2 + exit 61 + fi + done + if ! grep -Fq -- 'name = "trusted-workspace"' "$target_path/Cargo.toml"; then + echo "Error: Rust workflow context did not preserve trusted Cargo content ($target_path)" >&2 + exit 62 + fi + echo "scan ok with Rust workspace context" + exit 0 + ;; *) echo "unknown scenario ${FAKE_STRIX_SCENARIO:?}" >&2 exit 8 @@ -5332,6 +5445,18 @@ EOS touch "$repo_root_dir/docker-compose.yml" touch "$repo_root_dir/render.yaml" echo '0.0.0' >"$repo_root_dir/VERSION" + elif [ "$scenario" = "pr-rust-workspace-context" ]; then + mkdir -p "$repo_root_dir/.github/workflows" "$repo_root_dir/src" + echo 'name: Rust CI' >"$repo_root_dir/.github/workflows/rust.yml" + cat >"$repo_root_dir/Cargo.toml" <<'EOS' +[package] +name = "trusted-workspace" +version = "0.1.0" +EOS + echo '# trusted lock' >"$repo_root_dir/Cargo.lock" + echo '[toolchain]' >"$repo_root_dir/rust-toolchain.toml" + echo '[advisories]' >"$repo_root_dir/deny.toml" + echo 'fn main() {}' >"$repo_root_dir/src/main.rs" elif [ "$scenario" = "github-models-fallback-dockerfile-test-baseline-before-next-success-continues" ]; then mkdir -p "$repo_root_dir/.github/workflows" cat >"$repo_root_dir/.github/workflows/build-ci-image.yml" <<'EOS' @@ -5415,6 +5540,10 @@ EOS for large_scope_index in $(seq 1 38); do printf 'file %s\n' "$large_scope_index" >"$repo_root_dir/backend/large-scope/file-$large_scope_index.py" done + elif [ "$scenario" = "scan-working-directory-isolated" ]; then + mkdir -p "$repo_root_dir/backend/app/pg_introspect" + printf '%s\n' 'HEAD_INTROSPECT_SHOULD_BE_SCANNED' >"$repo_root_dir/backend/app/pg_introspect/introspect.py" + printf '%s\n' 'TRUSTED_DSN_GUARD_CONTEXT_SHOULD_BE_SCANNED' >"$repo_root_dir/backend/app/pg_introspect/dsn_guard.py" fi local scenario_base_sha="" @@ -5687,6 +5816,14 @@ PY "$repo_root_dir/strix_runs/fake-known-internal-warning/strix.log" \ "finish_scan: completed scan with 0 vulnerability report(s)" \ "scenario=$scenario keeps non-warning Strix report evidence" + assert_file_not_contains \ + "$repo_root_dir/strix_runs/fake-known-internal-warning-relative/strix.log" \ + "produced non-lifecycle final output" \ + "scenario=$scenario sanitizes relative scanner output before publication" + assert_file_contains \ + "$repo_root_dir/strix_runs/fake-known-internal-warning-relative/strix.log" \ + "finish_scan: completed scan with 0 vulnerability report(s)" \ + "scenario=$scenario publishes sanitized relative scanner evidence" assert_file_contains \ "$repo_root_dir/outside-strix-report/strix.log" \ "outside report should not be rewritten" \ @@ -5760,6 +5897,45 @@ run_gate_case_allow_provider_signal() { run_gate_case_with_provider_signal_mode "0" "$@" } +run_github_models_http410_case() { + local scenario="$1" + local expected_exit="$2" + local expected_calls="$3" + local expected_models="$4" + local expected_api_bases="$5" + local expected_message="${6-}" + + run_gate_case "$scenario" \ + "openai/gpt-5" \ + "" \ + "$expected_exit" \ + "$expected_message" \ + "$expected_calls" \ + "$expected_models" \ + "$expected_api_bases" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "" \ + "" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "deepseek/deepseek-r1-0528" \ + "1" +} + run_filtered_gate_case_if_requested() { case "${STRIX_TEST_CASE_FILTER:-}" in "") @@ -5775,6 +5951,28 @@ run_filtered_gate_case_if_requested() { "vertex_ai/ready-primary" \ "" ;; + pr-rust-workspace-context) + run_gate_case "pr-rust-workspace-context" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with Rust workspace context" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + ".github/workflows/rust.yml" + ;; success-with-critical-report) run_gate_case "success-with-critical-report" \ "vertex_ai/ready-primary" \ @@ -6094,6 +6292,23 @@ run_filtered_gate_case_if_requested() { "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" ;; + github-models-http410-authenticated-fallback-success) + run_github_models_http410_case \ + "$STRIX_TEST_CASE_FILTER" \ + "0" \ + "2" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ + "https://models.github.ai/inference|https://models.github.ai/inference" \ + "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." + ;; + github-models-http410-missing-http-token | github-models-http410-missing-provider-error | github-models-http410-numeric-continuation-4100 | github-models-http410-numeric-continuation-4104 | github-models-http410-target-output-spoof | github-models-retirement-brownout-phrase-only) + run_github_models_http410_case \ + "$STRIX_TEST_CASE_FILTER" \ + "1" \ + "1" \ + "openai/gpt-5" \ + "https://models.github.ai/inference" + ;; github-models-fallback-provider-signal-tries-next) run_gate_case "github-models-fallback-provider-signal-tries-next" \ "openai/gpt-5" \ @@ -6135,6 +6350,39 @@ run_filtered_gate_case_if_requested() { "vertex_ai/excluded-dir-primary" \ "" ;; + pull-request-target-changed-backend-context) + run_pull_request_target_changed_backend_context_scope_case + ;; + report-known-internal-warning-sanitized) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/report-known-internal-warning-sanitized" \ + "" \ + "0" \ + "Strix run succeeded for model 'vertex_ai/report-known-internal-warning-sanitized'" \ + "1" \ + "vertex_ai/report-known-internal-warning-sanitized" \ + "" + ;; + provider-fatal-success-signal | provider-warning-success-signal) + run_gate_case "$STRIX_TEST_CASE_FILTER" \ + "vertex_ai/$STRIX_TEST_CASE_FILTER" \ + "" \ + "1" \ + "Strix run emitted provider infrastructure or failure-signal output; failing closed." \ + "1" \ + "vertex_ai/$STRIX_TEST_CASE_FILTER" \ + "" + ;; + provider-report-rate-limit-fallback-success) + run_gate_case "provider-report-rate-limit-fallback-success" \ + "vertex_ai/report-rate-limit-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ + "2" \ + "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ + "|" + ;; total-timeout) run_total_timeout_case ;; @@ -6169,6 +6417,37 @@ run_filtered_gate_case_if_requested() { "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" ;; + github-models-exhausted-after-baseline-vulnerability-fails-closed) + run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ + "openai/gpt-5" \ + "" \ + "1" \ + "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ + "3" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ + "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ + "1" + ;; github-models-fallback-changed-vulnerability-before-next-success-blocks) run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ "openai/gpt-5" \ @@ -6298,6 +6577,28 @@ run_filtered_gate_case_if_requested() { "Materialized PR-head changed-file scope" \ "repository_dispatch" ;; + scan-working-directory-isolated) + run_gate_case "scan-working-directory-isolated" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with isolated Strix working directory" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "backend/app/pg_introspect/introspect.py" + ;; *) record_failure "unknown STRIX_TEST_CASE_FILTER '${STRIX_TEST_CASE_FILTER:-}'" ;; @@ -6982,6 +7283,34 @@ if [ -f "$target_path/backend/services/email_parser.py" ]; then matched_backend_context=1 fi +if [ -f "$target_path/backend/app/knowledge_graph.py" ]; then + if [ ! -f "$target_path/backend/app/post_eligibility.py" ]; then + echo "Error: backend/app local import context missing from PR scope ($target_path)" >&2 + exit 78 + fi + if ! grep -Fq -- 'BASE_POST_ELIGIBILITY_SHOULD_BE_SCANNED' "$target_path/backend/app/post_eligibility.py"; then + echo "Error: backend/app dependency context did not use trusted base content" >&2 + cat -- "$target_path/backend/app/post_eligibility.py" >&2 + exit 79 + fi + echo "scan ok with backend/app local import context" + matched_backend_context=1 +fi + +if [ -f "$target_path/contextual_orchestrator/__main__.py" ]; then + if [ ! -f "$target_path/contextual_orchestrator/cost_ledger.py" ]; then + echo "Error: contextual-orchestrator local import context missing from PR scope ($target_path)" >&2 + exit 80 + fi + if ! grep -Fq -- 'BASE_COST_LEDGER_SHOULD_BE_SCANNED' "$target_path/contextual_orchestrator/cost_ledger.py"; then + echo "Error: contextual-orchestrator dependency context did not use trusted base content" >&2 + cat -- "$target_path/contextual_orchestrator/cost_ledger.py" >&2 + exit 81 + fi + echo "scan ok with contextual-orchestrator local import context" + matched_backend_context=1 +fi + if [ "$matched_backend_context" -eq 1 ]; then exit 0 fi @@ -7005,6 +7334,9 @@ EOF printf '%s\n' 'BASE_EMAILS_CONTENT_SHOULD_NOT_BE_SCANNED' >backend/api/emails.py printf '%s\n' 'BASE_CALENDAR_SERVICE_SHOULD_BE_SCANNED' >backend/services/calendar_service.py printf '%s\n' 'BASE_LLM_PROVIDER_URLS_SHOULD_NOT_BE_SCANNED' >backend/services/llm_provider_urls.py + printf '%s\n' 'BASE_POST_ELIGIBILITY_SHOULD_BE_SCANNED' >backend/app/post_eligibility.py + mkdir -p contextual_orchestrator + printf '%s\n' 'BASE_COST_LEDGER_SHOULD_BE_SCANNED' >contextual_orchestrator/cost_ledger.py git add . git commit -qm 'base commit' ) @@ -7053,6 +7385,14 @@ EOF cat >backend/api/runner_config.py <<'EOF' def require_workspace_admin(): return 'HEAD_RUNNER_CONFIG_SHOULD_BE_SCANNED' +EOF + cat >backend/app/knowledge_graph.py <<'EOF' +from .post_eligibility import SOURCE_POST_ELIGIBILITY_SQL +HEAD_KNOWLEDGE_GRAPH_SHOULD_BE_SCANNED +EOF + cat >contextual_orchestrator/__main__.py <<'EOF' +from .cost_ledger import UsageRecord +HEAD_CONTEXTUAL_ORCHESTRATOR_SHOULD_BE_SCANNED EOF git add . git commit -qm 'head commit' @@ -7070,7 +7410,7 @@ EOF STRIX_INPUT_FILE_ROOT="$tmp_dir" \ GITHUB_EVENT_NAME="pull_request_target" \ PR_BASE_SHA="$base_sha" \ - PR_HEAD_SHA="$head_sha" \ + PR_HEAD_SHA=" $head_sha " \ STRIX_DISABLE_PR_SCOPING="0" \ FAKE_STRIX_CALL_LOG="$call_log" \ STRIX_LLM_FILE="$strix_llm_file" \ @@ -7087,6 +7427,8 @@ EOF assert_file_contains "$output_log" "scan ok with PR-head backend dependency context" "case=pull-request-target-changed-backend-context-uses-head-blob output" assert_file_contains "$output_log" "scan ok with PR-head LLM provider URL validation context" "case=pull-request-target-changed-backend-context-includes-llm-provider-url-validation output" assert_file_contains "$output_log" "scan ok with PR-head email parser text safety context" "case=pull-request-target-changed-backend-context-includes-email-parser-text-safety output" + assert_file_contains "$output_log" "scan ok with backend/app local import context" "case=pull-request-target-changed-backend-context-includes-backend-app-local-import output" + assert_file_contains "$output_log" "scan ok with contextual-orchestrator local import context" "case=pull-request-target-changed-contextual-orchestrator-includes-local-import output" assert_equals "1" "$(wc -l <"$call_log" | tr -d ' ')" "case=pull-request-target-changed-backend-context-uses-head-blob strix call count" rm -rf "$tmp_dir" @@ -8903,6 +9245,8 @@ assert_strix_workflow_pr_trigger_hardened assert_strix_pr_scope_includes_deployment_context +assert_strix_pr_scope_includes_contextual_orchestrator_context + assert_strix_gpt54_model_guard_cases assert_strix_gate_target_scope_separated @@ -9508,6 +9852,29 @@ run_gate_case_allow_provider_signal "github-models-primary-denied-fallback-succe "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" +run_github_models_http410_case \ + "github-models-http410-authenticated-fallback-success" \ + "0" \ + "2" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528" \ + "https://models.github.ai/inference|https://models.github.ai/inference" \ + "REGEX:Strix quick scan succeeded with fallback model 'deepseek/deepseek-r1-0528' in [0-9]+s\\." + +for scenario in \ + github-models-http410-missing-http-token \ + github-models-http410-missing-provider-error \ + github-models-http410-numeric-continuation-4100 \ + github-models-http410-numeric-continuation-4104 \ + github-models-http410-target-output-spoof \ + github-models-retirement-brownout-phrase-only; do + run_github_models_http410_case \ + "$scenario" \ + "1" \ + "1" \ + "openai/gpt-5" \ + "https://models.github.ai/inference" +done + run_gate_case "github-models-primary-ratelimit-fallback-success" \ "openai/gpt-5" \ "" \ @@ -9598,6 +9965,36 @@ run_gate_case "github-models-fallback-baseline-vulnerability-before-next-success "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ "1" +run_gate_case "github-models-exhausted-after-baseline-vulnerability-fails-closed" \ + "openai/gpt-5" \ + "" \ + "1" \ + "STRIX_PROVIDER_UNAVAILABLE: provider models were exhausted after incomplete scan evidence." \ + "3" \ + "openai/gpt-5|openai/deepseek/deepseek-r1-0528|openai/deepseek/deepseek-v3-0324" \ + "https://models.github.ai/inference|https://models.github.ai/inference|https://models.github.ai/inference" \ + "openai" \ + "https://models.github.ai/inference" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" \ + "" \ + "" \ + "0" \ + "" \ + "" \ + "" \ + "__SAME_AS_FALLBACK_MODELS__" \ + "deepseek/deepseek-r1-0528 deepseek/deepseek-v3-0324" \ + "1" + run_gate_case "github-models-fallback-changed-vulnerability-before-next-success-blocks" \ "openai/gpt-5" \ "" \ @@ -9993,6 +10390,15 @@ run_gate_case "provider-warning-success-signal" \ "" \ "1" +run_gate_case "provider-report-rate-limit-fallback-success" \ + "vertex_ai/report-rate-limit-primary" \ + "vertex_ai/fallback-one vertex_ai/fallback-two" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ + "2" \ + "vertex_ai/report-rate-limit-primary|vertex_ai/fallback-one" \ + "|" + run_gate_case "report-known-internal-warning-sanitized" \ "vertex_ai/report-known-internal-warning-sanitized" \ "" \ @@ -10769,6 +11175,27 @@ run_gate_case "pr-changed-scope-bounded" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" +run_gate_case "scan-working-directory-isolated" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with isolated Strix working directory" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + "backend/app/pg_introspect/introspect.py" + run_gate_case "pr-python-scope-context" \ "openai/gpt-4o-mini" \ "" \ @@ -10929,6 +11356,27 @@ run_gate_case "pr-deployment-scope-entrypoint-context" \ "pull_request" \ ".github/workflows/opencode-review.yml" +run_gate_case "pr-rust-workspace-context" \ + "openai/gpt-4o-mini" \ + "" \ + "0" \ + "scan ok with Rust workspace context" \ + "1" \ + "openai/gpt-4o-mini" \ + "https://example.invalid" \ + "vertex_ai" \ + "__DEFAULT__" \ + "" \ + "0" \ + "CRITICAL" \ + "0" \ + "" \ + "" \ + "1200" \ + "0" \ + "pull_request" \ + ".github/workflows/rust.yml" + run_gate_case "pr-empty-diff-skip" \ "openai/gpt-4o-mini" \ "" \ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index d7d0ec8ac..e58f5e6c0 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -45,6 +45,35 @@ def test_merge_scheduler_dispatches_one_review_by_default() -> None: ) +def test_organization_readiness_does_not_echo_untrusted_http_method( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Keep arbitrary HTTP method text out of organization-loop diagnostics.""" + from types import SimpleNamespace + + from scripts.ci.organization_commercial_readiness_loop import ( + GitHubClient, + GitHubError, + ) + + token = "ghp_abcdefghijklmnopqrstuvwxyz0123456789AB" + monkeypatch.setattr( + "subprocess.run", + lambda *_args, **_kwargs: SimpleNamespace( + returncode=1, + stdout="", + stderr="request rejected", + ), + ) + + with pytest.raises(GitHubError) as raised: + GitHubClient("client-token").request("/repos/example", method=token) + + message = str(raised.value) + assert token.upper() not in message + assert "[REDACTED_METHOD]" in message + + def test_merge_scheduler_rejects_untrusted_stale_timeout_values() -> None: """Dispatch payloads must not smuggle shell syntax into scheduler arguments.""" workflow = workflow_text("pr-review-merge-scheduler.yml") @@ -1479,19 +1508,25 @@ def test_optional_strix_workflow_absence_is_logged_without_failing_lookup() -> N assert 'if target_workflow_available "strix.yml"; then' in failed_check_evidence -def test_strix_provider_outage_without_findings_is_neutralized() -> None: - """Keep provider outages non-blocking only when no vulnerability finding exists.""" +def test_strix_provider_outage_without_findings_is_typed_non_passing() -> None: + """Keep provider outages typed and non-passing until authoritative evidence exists.""" workflow = workflow_text("strix.yml") assert "RateLimitError|Too many requests" in workflow assert "exceeded your current quota" in workflow assert "billing details" in workflow assert "LLM warm-up failed" in workflow + assert "model_behavior_error_signal=" in workflow + assert "agents|pydantic_ai|strix" in workflow assert "zero_vulnerabilities_signal" not in workflow + assert "Vulnerabilities[[:space:]]+[1-9]" in workflow assert "(^|[^A-Za-z0-9_])severity[[:space:]]*:" in workflow assert "STRIX_FAIL_ON_MIN_SEVERITY: MEDIUM" in workflow - assert "before producing a vulnerability report" in workflow - assert "genuine findings still fail the check" in workflow + assert "::error title=STRIX_PROVIDER_UNAVAILABLE::" in workflow + assert 'exit "$strix_rc"' in workflow + assert "Treating as a neutral skip" not in workflow + assert "authoritative vulnerability analysis" in workflow + assert "incomplete scan into passing security evidence" in workflow assert ( '&& ! grep -Eiq "$reported_vulnerability_signal" ' '"$strix_neutralization_scope_log"' in workflow diff --git a/tests/test_strix_backend_unavailable_after_exempted_finding.py b/tests/test_strix_backend_unavailable_after_exempted_finding.py index 3a087be07..3355a8448 100644 --- a/tests/test_strix_backend_unavailable_after_exempted_finding.py +++ b/tests/test_strix_backend_unavailable_after_exempted_finding.py @@ -1,4 +1,4 @@ -"""Regression contract for backend-outage neutral-skip after an exempted finding. +"""Regression contract for typed backend failure after an exempted finding. The Strix required check's console log can legitimately contain an already-exempted vulnerability (out-of-scope unchanged-file evidence, or one @@ -11,9 +11,9 @@ Before this fix, the workflow's outer neutral-skip decision grepped the whole combined log for `reported_vulnerability_signal`, so the earlier -- already exempted -- finding's own "Vulnerabilities N" / "severity:" text permanently -disqualified the neutral skip, turning a pure CI-infrastructure outage into a -required-check failure that blocks merges. The fix scopes that decision to -the log tail after the last "allowing pipeline continuation" marker. This +disqualified precise provider-failure classification. The fix scopes that +decision to the log tail after the last "allowing pipeline continuation" +marker while preserving a non-passing result for the incomplete scan. This test extracts the actual bash block from the workflow (not a reimplementation) and executes it against synthetic logs shaped like the real PR #392 run. """ @@ -66,18 +66,22 @@ def _extract_neutralization_block(workflow: str) -> str: start_marker = ( " # Recognized signals that the LLM backend was unavailable" ) + terminal_failure_marker = ( + ' echo "Strix reported security findings or failed for a ' + 'non-backend reason; failing the required check' + ) end_marker = ' exit "$strix_rc"\n' start = workflow.index(start_marker) - end = workflow.index(end_marker, start) + len(end_marker) + terminal_failure = workflow.index(terminal_failure_marker, start) + end = workflow.index(end_marker, terminal_failure) + len(end_marker) return workflow[start:end] def _run_gate_tail(log_text: str) -> int: """Execute the extracted block against a synthetic log; return its exit code. - 0 means the run neutral-skips (CI-infrastructure outage, not a finding). - Any other code means the block falls through to the hard failure branch, - matching the real workflow's `exit "$strix_rc"`. + A non-zero code is required because provider failure produced no + authoritative complete vulnerability result. """ workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") @@ -118,14 +122,14 @@ def test_workflow_defines_the_tail_scoping_step(self) -> None: workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") self.assertIn("strix_neutralization_scope_log", workflow) self.assertIn("allowing pipeline continuation", workflow) - self.assertIn("github_models_retirement_brownout", workflow) - self.assertIn("Error code:[[:space:]]*410", workflow) + self.assertIn("::error title=STRIX_PROVIDER_UNAVAILABLE::", workflow) + self.assertNotIn("Treating as a neutral skip", workflow) - def test_neutralizes_brownout_after_an_already_exempted_finding(self) -> None: - """The PR #392 shape: exempted finding, then an unrelated 410 brownout.""" + def test_brownout_after_an_already_exempted_finding_is_non_passing(self) -> None: + """The PR #392 shape remains typed and non-passing after an exemption.""" log = EXEMPTED_FINDING_AND_CONTINUATION + GITHUB_MODELS_BROWNOUT - self.assertEqual(_run_gate_tail(log), 0) + self.assertEqual(_run_gate_tail(log), 1) def test_still_fails_closed_on_a_finding_reported_after_continuation(self) -> None: """A real finding surfacing *after* the continuation marker still blocks.""" @@ -134,20 +138,20 @@ def test_still_fails_closed_on_a_finding_reported_after_continuation(self) -> No EXEMPTED_FINDING_AND_CONTINUATION + "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n" ) - self.assertNotEqual(_run_gate_tail(log), 0) + self.assertEqual(_run_gate_tail(log), 1) def test_still_fails_closed_with_no_continuation_marker_at_all(self) -> None: """Preserve prior behavior: a bare unresolved finding still blocks.""" log = "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n" - self.assertNotEqual(_run_gate_tail(log), 0) + self.assertEqual(_run_gate_tail(log), 1) - def test_still_neutralizes_a_bare_backend_outage_with_no_finding_at_all( + def test_bare_backend_outage_with_no_finding_is_non_passing( self, ) -> None: - """Preserve prior behavior: a pure outage with no finding still skips.""" + """A pure outage still lacks authoritative scan evidence.""" - self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 0) + self.assertEqual(_run_gate_tail(GITHUB_MODELS_BROWNOUT), 1) if __name__ == "__main__": diff --git a/tests/test_strix_local_proxy_bootstrap_failure_is_neutral.py b/tests/test_strix_local_proxy_bootstrap_failure_is_classified.py similarity index 81% rename from tests/test_strix_local_proxy_bootstrap_failure_is_neutral.py rename to tests/test_strix_local_proxy_bootstrap_failure_is_classified.py index c85d115e4..ea1f6517e 100644 --- a/tests/test_strix_local_proxy_bootstrap_failure_is_neutral.py +++ b/tests/test_strix_local_proxy_bootstrap_failure_is_classified.py @@ -7,7 +7,8 @@ failure-signal output; failing closed." (scripts/ci/strix_quick_gate.sh's `run_current_target_scan`, no fallback attempted because `is_model_retryable_error` doesn't recognize a local proxy-login failure as -an LLM-provider error). Before this fix, the workflow's neutral-skip regex +an LLM-provider error). Before this fix, the workflow's provider-failure +classification regex only matched the "emitted ..." wording variant of that message family, so this specific "scan failed after ..." wording fell through to a hard required-check failure even though zero vulnerabilities were reported. @@ -16,8 +17,8 @@ 97019252804): `loginAsGuest failed after 10 attempts: curl exit 7: ... Failed to connect to 127.0.0.1 port 48080`, "Vulnerabilities 0", then "Strix scan failed after provider infrastructure or failure-signal output; -failing closed." -- a pure CI-infrastructure hiccup that still failed the -required check. +failing closed." -- a pure CI-infrastructure hiccup. Classification is +diagnostic only: the incomplete scan must still fail the required check. """ from __future__ import annotations @@ -59,8 +60,8 @@ def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: return match.group(1) -def _workflow_neutralizes(log_text: str) -> bool: - """Execute the outer workflow's backend-neutralization condition.""" +def _workflow_classifies_provider_failure(log_text: str) -> bool: + """Evaluate the outer workflow's provider-failure classification inputs.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") backend_pattern = _workflow_signal_pattern(workflow, "backend_unavailable_signal") @@ -92,18 +93,15 @@ def _workflow_neutralizes(log_text: str) -> bool: class StrixLocalProxyBootstrapFailureTests(unittest.TestCase): """Protect the PR #392-shaped local-proxy failure without weakening the gate.""" - def test_workflow_recognizes_the_scan_failed_after_wording_variant(self) -> None: + def test_workflow_recognizes_the_authenticated_caido_failure_shape(self) -> None: workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") - self.assertIn("provider infrastructure or failure-signal output", workflow) - # The narrower "emitted ..." wording must not have silently regressed - # back in as the only recognized variant. - self.assertNotIn( - "emitted provider infrastructure or failure-signal output", - workflow, - ) + self.assertIn("Error during penetration test: loginAsGuest failed after", workflow) + self.assertIn("Failed to connect to 127\\.0\\.0\\.1 port 48080", workflow) - def test_neutralizes_local_proxy_bootstrap_failure_with_zero_findings(self) -> None: - self.assertTrue(_workflow_neutralizes(LOCAL_PROXY_BOOTSTRAP_FAILURE)) + def test_classifies_local_proxy_bootstrap_failure_with_zero_findings(self) -> None: + self.assertTrue( + _workflow_classifies_provider_failure(LOCAL_PROXY_BOOTSTRAP_FAILURE) + ) def test_still_fails_closed_when_a_real_vulnerability_is_also_reported( self, @@ -111,7 +109,7 @@ def test_still_fails_closed_when_a_real_vulnerability_is_also_reported( log = LOCAL_PROXY_BOOTSTRAP_FAILURE + ( "Vulnerability Report\nSeverity: CRITICAL\nVulnerabilities 1\n" ) - self.assertFalse(_workflow_neutralizes(log)) + self.assertFalse(_workflow_classifies_provider_failure(log)) if __name__ == "__main__": diff --git a/tests/test_strix_model_behavior_error.py b/tests/test_strix_model_behavior_error.py new file mode 100644 index 000000000..0918be59f --- /dev/null +++ b/tests/test_strix_model_behavior_error.py @@ -0,0 +1,226 @@ +"""Regression contract for Strix ModelBehaviorError protocol flakes. + +A ModelBehaviorError with zero reported vulnerabilities is retryable model +evidence. Real vulnerability counts remain fail-closed. +""" + +from __future__ import annotations + +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +STRIX_GATE = REPOSITORY_ROOT / "scripts" / "ci" / "strix_quick_gate.sh" +STRIX_WORKFLOW = REPOSITORY_ROOT / ".github" / "workflows" / "strix.yml" +QUALITY_WORKFLOW = ( + REPOSITORY_ROOT / ".github" / "workflows" / "strix-changed-path-quality-ci.yml" +) + + +def _function_block(source: str, function_name: str) -> str: + """Return one top-level Bash function, including its closing brace.""" + + match = re.search( + rf"(?ms)^{re.escape(function_name)}\(\) {{\n.*?^}}\n", + source, + ) + if match is None: + raise AssertionError(f"missing Bash function: {function_name}") + return match.group(0) + + +def _classifies_as_model_behavior_error(log_text: str) -> bool: + """Execute the production classifier against a bounded synthetic log.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + function_source = _function_block(gate_source, "is_model_behavior_error") + with tempfile.TemporaryDirectory(prefix="strix-model-behavior-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + script = "\n".join( + ( + "set -euo pipefail", + 'STRIX_LOG="$1"', + function_source, + "is_model_behavior_error", + ) + ) + completed = subprocess.run( + ["bash", "-c", script, "strix-classifier", str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode not in {0, 1}: + raise AssertionError(completed.stderr) + return completed.returncode == 0 + + +def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: + """Extract one single-quoted POSIX ERE assigned in the Strix workflow.""" + + match = re.search( + rf"(?m)^\s+{re.escape(variable_name)}='([^']+)'$", + workflow, + ) + if match is None: + raise AssertionError(f"missing workflow signal: {variable_name}") + return match.group(1) + + +def _workflow_neutralizes(log_text: str) -> bool: + """Execute the outer workflow's backend-neutralization condition.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + backend_pattern = _workflow_signal_pattern( + workflow, + "backend_unavailable_signal", + ) + model_behavior_pattern = _workflow_signal_pattern( + workflow, + "model_behavior_error_signal", + ) + vulnerability_pattern = _workflow_signal_pattern( + workflow, + "reported_vulnerability_signal", + ) + with tempfile.TemporaryDirectory(prefix="strix-workflow-mbe-") as temp_dir: + log_path = Path(temp_dir) / "strix.log" + log_path.write_text(log_text, encoding="utf-8") + backend = subprocess.run( + ["grep", "-Eiq", backend_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + model_behavior = subprocess.run( + ["grep", "-Eq", model_behavior_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + vulnerability = subprocess.run( + ["grep", "-Eiq", vulnerability_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) + if backend.returncode not in {0, 1}: + raise AssertionError(backend.stderr) + if model_behavior.returncode not in {0, 1}: + raise AssertionError(model_behavior.stderr) + if vulnerability.returncode not in {0, 1}: + raise AssertionError(vulnerability.stderr) + return ( + (backend.returncode == 0 or model_behavior.returncode == 0) + and vulnerability.returncode == 1 + ) + + +class StrixModelBehaviorErrorTests(unittest.TestCase): + """Protect protocol flakes without weakening vulnerability fail-closed.""" + + def test_runtime_model_behavior_error_is_retryable(self) -> None: + """Recognize the exact PascalCase Strix agent-protocol exception.""" + + log = ( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) + + def test_lowercase_application_prose_is_not_retryable(self) -> None: + """Reject target-application text that only resembles the exception.""" + + log = "the model behavior error was logged by the scanned service\n" + self.assertFalse(_classifies_as_model_behavior_error(log)) + self.assertFalse(_classifies_as_model_behavior_error("ModelBehaviorError\n")) + + def test_agents_sdk_tool_protocol_failure_is_retryable(self) -> None: + """Recognize the OpenAI Agents SDK exception observed in required CI.""" + + log = ( + "agents.exceptions.ModelBehaviorError: Tool ls not found in agent strix\n" + "Vulnerabilities 0\n" + ) + self.assertTrue(_classifies_as_model_behavior_error(log)) + + def test_behavior_error_skips_same_model_and_enters_fallback(self) -> None: + """Wire the classifier into infrastructure and cross-model fallback.""" + + gate_source = STRIX_GATE.read_text(encoding="utf-8") + infrastructure = _function_block( + gate_source, + "has_detected_infrastructure_error", + ) + retryable = _function_block(gate_source, "is_model_retryable_error") + same_model_retry = _function_block( + gate_source, + "is_transient_same_model_retry_error", + ) + + self.assertIn("is_model_behavior_error", infrastructure) + self.assertIn("is_model_behavior_error", retryable) + self.assertNotIn("is_model_behavior_error", same_model_retry) + + def test_outer_workflow_classifies_zero_finding_protocol_flake(self) -> None: + """Empty scans that hit ModelBehaviorError receive typed diagnostics.""" + + self.assertTrue( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 0\n" + ) + ) + self.assertFalse( + _workflow_neutralizes("ModelBehaviorError\nVulnerabilities 0\n") + ) + self.assertFalse( + _workflow_neutralizes( + "agents.foo.modelbehaviorerror\nVulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: + """Keep a real vulnerability signal blocking despite protocol failure.""" + + self.assertFalse( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 1\n" + ) + ) + self.assertFalse( + _workflow_neutralizes( + "strix.agents.base.ModelBehaviorError: tool protocol mismatch\n" + "Vulnerabilities 9\n" + ) + ) + + def test_workflow_keeps_fail_closed_vulnerability_contract(self) -> None: + """Retain the static fail-closed vulnerability evidence contract.""" + + workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") + self.assertIn("ModelBehaviorError", workflow) + self.assertIn("model_behavior_error_signal", workflow) + self.assertIn("reported_vulnerability_signal", workflow) + self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn( + '! grep -Eiq "$reported_vulnerability_signal"', + workflow, + ) + + def test_quality_trigger_includes_model_behavior_contracts(self) -> None: + """Keep classifier, doctoring, and workflow edits on the quality path.""" + + workflow = QUALITY_WORKFLOW.read_text(encoding="utf-8") + self.assertIn(' - "docs/doctoring/strix-model-behavior-error.md"', workflow) + self.assertIn(' - "tests/test_strix_model_behavior_error.py"', workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_strix_nvidia_nim_not_found_fallback.py b/tests/test_strix_nvidia_nim_not_found_fallback.py index dd1bc3132..990269725 100644 --- a/tests/test_strix_nvidia_nim_not_found_fallback.py +++ b/tests/test_strix_nvidia_nim_not_found_fallback.py @@ -85,7 +85,7 @@ def _workflow_signal_pattern(workflow: str, variable_name: str) -> str: return match.group(1) -def _workflow_neutralizes(log_text: str) -> bool: +def _workflow_classifies_backend_unavailable(log_text: str) -> bool: """Execute the outer workflow's backend-neutralization condition.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") @@ -93,6 +93,10 @@ def _workflow_neutralizes(log_text: str) -> bool: workflow, "backend_unavailable_signal", ) + model_behavior_pattern = _workflow_signal_pattern( + workflow, + "model_behavior_error_signal", + ) vulnerability_pattern = _workflow_signal_pattern( workflow, "reported_vulnerability_signal", @@ -106,6 +110,12 @@ def _workflow_neutralizes(log_text: str) -> bool: capture_output=True, text=True, ) + model_behavior = subprocess.run( + ["grep", "-Eq", model_behavior_pattern, str(log_path)], + check=False, + capture_output=True, + text=True, + ) vulnerability = subprocess.run( ["grep", "-Eiq", vulnerability_pattern, str(log_path)], check=False, @@ -114,9 +124,14 @@ def _workflow_neutralizes(log_text: str) -> bool: ) if backend.returncode not in {0, 1}: raise AssertionError(backend.stderr) + if model_behavior.returncode not in {0, 1}: + raise AssertionError(model_behavior.stderr) if vulnerability.returncode not in {0, 1}: raise AssertionError(vulnerability.stderr) - return backend.returncode == 0 and vulnerability.returncode == 1 + return ( + (backend.returncode == 0 or model_behavior.returncode == 0) + and vulnerability.returncode == 1 + ) class StrixNvidiaNotFoundFallbackTests(unittest.TestCase): @@ -202,12 +217,12 @@ def test_outer_workflow_requires_litellm_context_for_nvidia_404(self) -> None: """Reject provider-like target text in the outer neutralization gate.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "source literal: Nvidia_nimException Error code: 404\n" ) ) self.assertTrue( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "litellm.exceptions.NotFoundError: Nvidia_nimException - " "Error code: 404\nVulnerabilities 0\n" ) @@ -217,7 +232,7 @@ def test_outer_workflow_rejects_cross_line_signal_assembly(self) -> None: """Require exception, provider, and 404 evidence on one physical line.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "litellm.exceptions.NotFoundError: provider unavailable\n" "Nvidia_nimException Error code: 404\n" ) @@ -227,22 +242,22 @@ def test_outer_workflow_rejects_nvidia_404_without_litellm_context(self) -> None """Require LiteLLM NotFoundError context, not just NVIDIA + 404.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "Nvidia_nimException Error code: 404\nVulnerabilities 0\n" ) ) - def test_outer_workflow_never_neutralizes_reported_vulnerabilities(self) -> None: + def test_outer_workflow_never_classifies_reported_vulnerabilities(self) -> None: """Keep a real vulnerability signal blocking despite provider failure.""" self.assertFalse( - _workflow_neutralizes( + _workflow_classifies_backend_unavailable( "litellm.exceptions.NotFoundError: Nvidia_nimException - " "Error code: 404\nVulnerabilities 1\n" ) ) - def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: + def test_workflow_classifies_backend_unavailable_only_nvidia_404_without_findings(self) -> None: """Retain the static fail-closed vulnerability evidence contract.""" workflow = STRIX_WORKFLOW.read_text(encoding="utf-8") @@ -250,10 +265,70 @@ def test_workflow_neutralizes_only_nvidia_404_without_findings(self) -> None: self.assertIn("Error code:[[:space:]]*404", workflow) self.assertIn("reported_vulnerability_signal", workflow) self.assertIn("Vulnerabilities[[:space:]]+[1-9]", workflow) + self.assertIn("model_behavior_error_signal=", workflow) + self.assertIn("agents|pydantic_ai|strix", workflow) self.assertIn( '! grep -Eiq "$reported_vulnerability_signal"', workflow, ) + self.assertIn("::error title=STRIX_PROVIDER_UNAVAILABLE::", workflow) + self.assertIn('exit "$strix_rc"', workflow) + self.assertNotIn("Treating as a neutral skip", workflow) + + def test_outer_workflow_classifies_backend_unavailable_model_behavior_error_without_findings( + self, + ) -> None: + """Require the actual scanner ModelBehaviorError format before classifying.""" + + self.assertFalse( + _workflow_classifies_backend_unavailable("ModelBehaviorError\nVulnerabilities 0\n") + ) + self.assertTrue( + _workflow_classifies_backend_unavailable( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_classifies_model_behavior_error_with_findings( + self, + ) -> None: + """Keep Vulnerabilities [1-9] fail-closed for the actual model exception.""" + + self.assertFalse( + _workflow_classifies_backend_unavailable( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 1\n" + ) + ) + + def test_outer_workflow_classifies_caido_bootstrap_failure_without_findings(self) -> None: + """Treat a Strix-owned Caido bootstrap outage as incomplete infrastructure evidence.""" + + self.assertTrue( + _workflow_classifies_backend_unavailable( + "Error during penetration test: loginAsGuest failed after 10 attempts: " + "curl exit 7: curl: (7) Failed to connect to 127.0.0.1 port 48080\n" + "Vulnerabilities 0\n" + ) + ) + + def test_outer_workflow_never_downgrades_caido_failure_with_findings(self) -> None: + """Keep a real finding blocking even when the Strix container also failed.""" + + self.assertFalse( + _workflow_classifies_backend_unavailable( + "Error during penetration test: loginAsGuest failed after 10 attempts: " + "curl exit 7: curl: (7) Failed to connect to 127.0.0.1 port 48080\n" + "Vulnerabilities 1\n" + ) + ) + self.assertFalse( + _workflow_classifies_backend_unavailable( + "agents.exceptions.ModelBehaviorError: provider response failed\n" + "Vulnerabilities 9\n" + ) + ) if __name__ == "__main__": diff --git a/tests/test_strix_quality_timeout_fixture_budget.py b/tests/test_strix_quality_timeout_fixture_budget.py index 78fcc8a7a..0ea4e3b37 100644 --- a/tests/test_strix_quality_timeout_fixture_budget.py +++ b/tests/test_strix_quality_timeout_fixture_budget.py @@ -33,6 +33,8 @@ def test_strix_quality_trigger_includes_fixture_contract_paths() -> None: assert "docs/doctoring/strix-quality-timeout-fixtures.md" in trigger assert "tests/test_strix_quality_timeout_fixture_budget.py" in trigger + assert "docs/doctoring/strix-model-behavior-error.md" in trigger + assert "tests/test_strix_model_behavior_error.py" in trigger def test_strix_quality_keeps_real_scanner_budgets_out_of_fixture_overrides() -> None: