From 8f6d483ea3c25906db7f71ff956547b7eac3927c Mon Sep 17 00:00:00 2001 From: srt0422 Date: Mon, 25 May 2026 15:17:48 -0700 Subject: [PATCH 1/8] feat(workflows): add org-wide go.mod replace-directive audit (DEVOP-617) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a weekly + manual-dispatch audit that enumerates every Go module across the allora-network org, fetches each `go.mod` via the Contents API, extracts every `replace` directive (single-line + `replace (...)` block form) with the canonical awk extractor from shai-hulud-defense REFERENCE.md, and classifies the RHS against the same trusted-host allowlist that scripts/shai-hulud-ioc-sweep.sh uses. Findings: - SUSPICIOUS (RHS non-allowlisted host AND LHS module path != RHS) → rolling GitHub Issue (label `gomod-replace-audit`) + Slack page via SLACK_SECURITY_WEBHOOK. - legitimate-version-pin (LHS == RHS module path, structurally cannot redirect) → no-op. - Fetch failures → rolling-issue update only (operational, not IOC). Distinct rolling-issue label from DEVOP-560's `shai-hulud-sweep` so the two pipelines don't collide. SHA-pinned `uses:`. Permissions are `contents: read` + `issues: write` only. Initial point-in-time audit committed at docs/security/gomod-replace-audit-2026-05-25.md: 12 modules scanned, 6 replace directives, 0 SUSPICIOUS findings. All current replaces are same-path version pins from the Cosmos SDK simapp pattern (gin-gonic/gin, syndtr/goleveldb, cosmos/cosmos-sdk, cometbft/cometbft). Refs: https://linear.app/alloralabs/issue/DEVOP-617 Co-authored-by: Cursor --- .github/workflows/gomod-replace-audit.yml | 426 ++++++++++++++++++ ...026-05-25-devop-617-gomod-replace-audit.md | 31 ++ .../gomod-replace-audit-2026-05-25.md | 109 +++++ 3 files changed, 566 insertions(+) create mode 100644 .github/workflows/gomod-replace-audit.yml create mode 100644 docs/plans/2026-05-25-devop-617-gomod-replace-audit.md create mode 100644 docs/security/gomod-replace-audit-2026-05-25.md diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml new file mode 100644 index 0000000..9770b15 --- /dev/null +++ b/.github/workflows/gomod-replace-audit.yml @@ -0,0 +1,426 @@ +name: Go.mod Replace-Directive Audit + +# Weekly org-wide audit of `replace` directives in every allora-network Go +# module. Reference: DEVOP-617. Complements the deeper DEVOP-560 daily +# Shai-Hulud IOC sweep — this is a lighter no-clone Contents-API pass that +# can scale to all org repos on a weekly cadence without re-cloning them. +# +# Detects the Go-side Shai-Hulud vector: a `replace` directive whose RHS +# points outside the org's canonical trusted-host allowlist (a compromised +# go.mod can silently redirect a legitimate import to an attacker fork). +# +# Outputs: +# - clean run: no-op (no issue update, no Slack page). +# - non-allowlisted hit: append to / open rolling issue labelled +# `gomod-replace-audit` and (if configured) +# page Slack via `SLACK_SECURITY_WEBHOOK`. +# +# Humans drive close/reopen of the rolling issue so triage state survives +# weekly runs. + +on: + schedule: + # 05:17 UTC every Monday — off-peak + off-minute to dodge GitHub + # Actions cron contention spikes on whole-hour boundaries, and + # offset from the daily Shai-Hulud sweep (04:07 UTC) so the two + # don't compete for the same minute-zero scheduler slot. + - cron: '17 5 * * 1' + workflow_dispatch: + +permissions: + contents: read + issues: write + +# Serialize concurrent runs so a manual `workflow_dispatch` while the +# weekly cron is mid-audit can't race on the rolling issue / Slack page. +concurrency: + group: gomod-replace-audit + cancel-in-progress: false + +jobs: + audit: + name: Audit allora-network for suspicious go.mod replaces + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + ORG: allora-network + ROLLING_LABEL: gomod-replace-audit + # Canonical trusted-host allowlist. MUST match + # scripts/shai-hulud-ioc-sweep.sh#GO_TRUSTED_HOSTS and + # shai-hulud-defense REFERENCE.md §Go specifics — keep in lockstep. + # Boundary `(/|$)` prevents lookalike paths (e.g. + # `github.com/cosmosmalicious/...` matching `github.com/cosmos`). + GO_TRUSTED_HOSTS_RE: 'github\.com/(allora-network|cosmos|ethereum|fluxcd)|gopkg\.in|google\.golang\.org|go\.uber\.org|go\.opentelemetry\.io|k8s\.io|sigs\.k8s\.io' + # Prefer a dedicated org-read PAT/App token when provisioned — + # required to enumerate private repos. Falls back to the workflow's + # default GITHUB_TOKEN, which can only see public org repos; + # `gh search code` will simply omit private hits in that mode, so + # the partial-coverage state is visible in the audit log (no + # silent false-clean). Aligned with DEVOP-560's pattern. + GH_TOKEN: ${{ secrets.GH_ORG_READ_TOKEN || secrets.GITHUB_TOKEN }} + + steps: + - name: Checkout .github repo (allowlist is sourced from env above) + # SHA pin: actions/checkout v4.2.2 (Oct 2024). Matches the pin + # used by .github/workflows/shai-hulud-sweep.yml (DEVOP-560) and + # ci-workflows-private hardened reusable workflows. + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + persist-credentials: false + + - name: Verify required tools + shell: bash + run: | + set -euo pipefail + gh --version + jq --version + awk --version | head -1 || true + base64 --version | head -1 || true + + - name: Enumerate org Go modules + id: enumerate + shell: bash + env: + OUTPUT_DIR: ${{ runner.temp }}/gomod-audit + run: | + set -euo pipefail + mkdir -p "$OUTPUT_DIR" + # `gh search code` returns repo + file path tuples. Some repos + # have multiple go.mod files (monorepo sub-modules); the dedup + # happens on the `repopath` pair, not on `repo` alone. + gh search code --owner "$ORG" 'filename:go.mod' --limit 200 \ + --json repository,path \ + --jq '.[] | "\(.repository.nameWithOwner)\t\(.path)"' \ + | sort -u > "$OUTPUT_DIR/paths.tsv" + count="$(wc -l < "$OUTPUT_DIR/paths.tsv" | tr -d ' ')" + echo "Discovered $count tuples." + { + echo "paths_file=$OUTPUT_DIR/paths.tsv" + echo "output_dir=$OUTPUT_DIR" + echo "count=$count" + } >> "$GITHUB_OUTPUT" + + - name: Fetch and audit each go.mod + id: audit + shell: bash + env: + OUTPUT_DIR: ${{ steps.enumerate.outputs.output_dir }} + PATHS_FILE: ${{ steps.enumerate.outputs.paths_file }} + run: | + set -uo pipefail + mkdir -p "$OUTPUT_DIR/gomods" + : > "$OUTPUT_DIR/audit.tsv" + : > "$OUTPUT_DIR/fetch-failures.tsv" + printf 'repo\tpath\tline_no\tlhs\trhs\tclassification\toriginal_line\n' \ + > "$OUTPUT_DIR/audit.tsv" + + while IFS=$'\t' read -r repo path; do + [ -z "$repo" ] && continue + safe="$(printf '%s' "$repo/$path" | tr '/' '_')" + gomod="$OUTPUT_DIR/gomods/$safe" + # Use `gh api` to fetch the contents-API JSON, jq out the + # base64 `.content` field, decode. A missing/empty repo or + # a moved default branch is a fetch failure, not a clean + # audit — record it so it surfaces in the rolling issue. + if ! gh api "repos/$repo/contents/$path" --jq '.content' 2>/dev/null \ + | base64 -d > "$gomod" 2>/dev/null \ + || [ ! -s "$gomod" ]; then + printf '%s\t%s\n' "$repo" "$path" >> "$OUTPUT_DIR/fetch-failures.tsv" + rm -f "$gomod" + continue + fi + + # Awk extractor — same logic as shai-hulud-defense REFERENCE.md + # §Go specifics + scripts/shai-hulud-ioc-sweep.sh. Handles + # both `replace LHS => RHS vX` single-line form and + # `replace ( ... )` block form. Strips trailing version + # from RHS so the classification gate sees the bare module + # path. + awk -v repo="$repo" -v path="$path" ' + /^[[:space:]]*replace[[:space:]]*\(/ { inblock=1; next } + inblock && /^[[:space:]]*\)/ { inblock=0; next } + /^[[:space:]]*replace[[:space:]]/ || inblock { + n = index($0, "=>") + if (n == 0) next + lhs = substr($0, 1, n - 1) + rhs = substr($0, n + 2) + sub(/^[[:space:]]*replace[[:space:]]+/, "", lhs) + sub(/^[[:space:]]+/, "", lhs); sub(/[[:space:]]+$/, "", lhs) + sub(/[[:space:]]+v[0-9].*$/, "", lhs) + sub(/^[[:space:]]+/, "", rhs); sub(/[[:space:]]+v[0-9].*$/, "", rhs) + sub(/[[:space:]]+$/, "", rhs) + if (rhs == "") next + original=$0 + gsub(/\t/, " ", original) + printf "%s\t%s\t%d\t%s\t%s\t%s\n", repo, path, NR, lhs, rhs, original + } + ' "$gomod" \ + | while IFS=$'\t' read -r r p line_no lhs rhs orig; do + case "$rhs" in + ./*|../*) + class="legitimate-local-relative" ;; + /*) + class="investigate-absolute" ;; + *) + if printf '%s\n' "$rhs" | grep -qE "^($GO_TRUSTED_HOSTS_RE)(/|$)"; then + class="legitimate-allowlisted-host" + elif [ "$lhs" = "$rhs" ]; then + # Same-path version pin (LHS module path == RHS + # module path). Structurally cannot redirect to an + # attacker fork — the RHS is the identical upstream + # module, just forced to a specific version. Safe; + # the host-allowlist filter alone would over-flag + # this and bury real redirect signals. Distinct + # class so future audits don't re-litigate it. + class="legitimate-version-pin" + else + class="SUSPICIOUS" + fi + ;; + esac + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$r" "$p" "$line_no" "$lhs" "$rhs" "$class" "$orig" \ + >> "$OUTPUT_DIR/audit.tsv" + done + done < "$PATHS_FILE" + + # Counts. SUSPICIOUS + investigate-absolute are IOC-grade. + total=$(($(wc -l < "$OUTPUT_DIR/audit.tsv") - 1)) + suspicious=$(awk -F'\t' '$6=="SUSPICIOUS" || $6=="investigate-absolute"' \ + "$OUTPUT_DIR/audit.tsv" | wc -l | tr -d ' ') + fetch_fails=$(wc -l < "$OUTPUT_DIR/fetch-failures.tsv" | tr -d ' ') + { + echo "total_directives=$total" + echo "suspicious=$suspicious" + echo "fetch_failures=$fetch_fails" + echo "audit_file=$OUTPUT_DIR/audit.tsv" + echo "fetch_failures_file=$OUTPUT_DIR/fetch-failures.tsv" + } >> "$GITHUB_OUTPUT" + + { + echo "# go.mod replace-directive audit" + echo + echo "**Run:** $(date -u)" + echo "**Org:** $ORG" + echo "**Modules scanned:** ${{ steps.enumerate.outputs.count }}" + echo "**Replace directives found:** $total" + echo "**Fetch failures:** $fetch_fails" + echo "**SUSPICIOUS (incident-grade):** $suspicious" + echo + if [ "$suspicious" -gt 0 ]; then + echo "## SUSPICIOUS findings — INVOKE INCIDENT RESPONSE" + echo + echo '```' + awk -F'\t' 'NR==1 || $6=="SUSPICIOUS" || $6=="investigate-absolute"' \ + "$OUTPUT_DIR/audit.tsv" | column -t -s $'\t' + echo '```' + echo + fi + if [ "$total" -gt 0 ]; then + echo "## All replace directives (full audit)" + echo + echo '```' + column -t -s $'\t' < "$OUTPUT_DIR/audit.tsv" + echo '```' + fi + if [ "$fetch_fails" -gt 0 ]; then + echo + echo "## Fetch failures (operational — review)" + echo + echo '```' + cat "$OUTPUT_DIR/fetch-failures.tsv" + echo '```' + fi + } > "$OUTPUT_DIR/summary.md" + + echo "::group::Audit summary" + cat "$OUTPUT_DIR/summary.md" + echo "::endgroup::" + echo "summary_file=$OUTPUT_DIR/summary.md" >> "$GITHUB_OUTPUT" + + - name: Upload audit artifacts + if: always() && steps.audit.outputs.audit_file != '' + # SHA pin: actions/upload-artifact v4.4.3 (Oct 2024). Same pin + # as DEVOP-560 sweep workflow. + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: gomod-replace-audit-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ steps.audit.outputs.audit_file }} + ${{ steps.audit.outputs.fetch_failures_file }} + ${{ steps.audit.outputs.summary_file }} + if-no-files-found: warn + retention-days: 30 + + - name: Update rolling issue on any non-clean run + # Non-clean here means: SUSPICIOUS finding OR fetch failure. A + # clean audit (no findings, all repos fetched) leaves the issue + # alone. Humans drive close/reopen so triage state persists. + if: steps.audit.outputs.suspicious != '0' || steps.audit.outputs.fetch_failures != '0' + shell: bash + env: + SUMMARY_PATH: ${{ steps.audit.outputs.summary_file }} + SUSPICIOUS: ${{ steps.audit.outputs.suspicious }} + FETCH_FAILS: ${{ steps.audit.outputs.fetch_failures }} + # Issue ops scoped to THIS repo only — the workflow's default + # GITHUB_TOKEN with `issues: write` is sufficient. A wider + # GH_ORG_READ_TOKEN (used above for enumeration) may or may + # not include issues:write here; keep this scoped narrow. + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + ts="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + if [ "$SUSPICIOUS" != "0" ]; then + grade="SUSPICIOUS findings (incident-grade)" + else + grade="fetch failures only (operational)" + fi + { + echo "### Audit run — $ts" + echo + echo "- **Result:** $grade" + echo "- **SUSPICIOUS:** $SUSPICIOUS" + echo "- **Fetch failures:** $FETCH_FAILS" + echo "- **Run:** $run_url" + echo "- **Trigger:** \`${GITHUB_EVENT_NAME}\`" + echo + # Summary may contain attacker-controllable strings from + # scanned go.mod content (module paths, version strings). + # Strip characters that could escape the code fence or + # inject markdown/HTML; the uploaded audit.tsv artifact is + # the canonical un-sanitized source for forensic review. + echo '```' + tr -d '`<>|*_' < "$SUMMARY_PATH" + echo '```' + } > /tmp/audit-comment.md + + # Find oldest open rolling issue with our label (newest gets + # surfaced by default; use `--search` with `sort:created-asc` + # so a long-running triage thread stays canonical even if a + # duplicate gets filed). `gh issue list` does not expose + # `--sort`; the search query is the supported lever. + existing="$(gh issue list \ + --search "label:\"$ROLLING_LABEL\" state:open sort:created-asc" \ + --limit 1 \ + --json number \ + --jq '.[0].number // empty')" + + if [ -n "$existing" ]; then + echo "Appending to existing rolling issue #$existing" + gh issue comment "$existing" --body-file /tmp/audit-comment.md + else + echo "Creating new rolling issue (label=$ROLLING_LABEL)" + # `--force` upserts the label (idempotent — safe to run on + # every cold-start of the rolling-issue cycle). + gh label create "$ROLLING_LABEL" \ + --description "Weekly go.mod replace-directive audit findings (DEVOP-617)" \ + --color D93F0B \ + --force >/dev/null 2>&1 || true + title="[gomod-replace-audit] rolling findings — $(date -u +%Y-%m)" + gh issue create \ + --label "$ROLLING_LABEL" \ + --title "$title" \ + --body-file /tmp/audit-comment.md + fi + + - name: Page Slack on SUSPICIOUS findings + # Only SUSPICIOUS findings page Slack. Fetch failures only + # update the rolling issue — those are worth review but not + # incident-grade. Cleanly no-op if the webhook secret is + # unset (early exit) so an org without it doesn't get a red + # weekly workflow. + if: always() && steps.audit.outputs.suspicious != '0' + shell: bash + env: + SLACK_SECURITY_WEBHOOK: ${{ secrets.SLACK_SECURITY_WEBHOOK }} + SUMMARY_PATH: ${{ steps.audit.outputs.summary_file }} + SUSPICIOUS: ${{ steps.audit.outputs.suspicious }} + run: | + set -euo pipefail + if [ -z "${SLACK_SECURITY_WEBHOOK:-}" ]; then + echo "::warning::SLACK_SECURITY_WEBHOOK secret not set; skipping Slack page." + exit 0 + fi + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + # Cap to ~2.8KB so the Slack block stays inside the 3000-char + # per-block limit after JSON escaping. Strip characters that + # could escape the Slack code fence or inject block-kit mrkdwn. + # audit.tsv (preserved in the uploaded artifact) is the + # canonical un-sanitized source for forensic review. + summary="$(head -c 6000 "$SUMMARY_PATH" 2>/dev/null || echo 'summary unavailable')" + summary="$(printf '%s' "$summary" | tr -d '`<>|*_')" + payload="$(jq -nc \ + --arg run "$run_url" \ + --arg org "$ORG" \ + --arg n "$SUSPICIOUS" \ + --arg summary "$summary" \ + '{ + text: (":rotating_light: go.mod replace audit — " + $n + " SUSPICIOUS finding(s) in " + $org), + blocks: [ + { type: "header", + text: { type: "plain_text", + text: "go.mod replace-directive audit — SUSPICIOUS findings" } }, + { type: "section", + text: { type: "mrkdwn", + text: (":rotating_light: *" + $n + " SUSPICIOUS go.mod replace directive(s) — " + $org + "*\n<" + $run + "|Open workflow run>") } }, + { type: "section", + text: { type: "mrkdwn", + text: ("```\n" + ($summary | .[0:2800]) + "\n```") } } + ] + }')" + # Retry on transient HTTP failures (5s, 15s, 45s). Honor + # Slack's Retry-After on HTTP 429. Same retry shape as the + # DEVOP-560 sweep Slack page step. + http_code=000 + attempt=0 + max_attempts=3 + delays=(5 15 45) + while [ "$attempt" -lt "$max_attempts" ]; do + attempt=$((attempt + 1)) + http_code="$(curl -sS --connect-timeout 5 --max-time 15 \ + -X POST -H 'Content-Type: application/json' \ + --data "$payload" \ + -D /tmp/slack.headers \ + -o /tmp/slack.out \ + -w '%{http_code}' \ + "$SLACK_SECURITY_WEBHOOK" || echo 000)" + if [ "$http_code" = "200" ]; then + break + fi + case "$http_code" in + 000|408|429|5*) transient=1 ;; + *) transient=0 ;; + esac + if [ "$transient" -ne 1 ] || [ "$attempt" -ge "$max_attempts" ]; then + break + fi + sleep_for="${delays[$((attempt - 1))]}" + retry_after="$(awk 'BEGIN{IGNORECASE=1} /^Retry-After:/ {gsub(/[\r\n]/,"",$2); print $2; exit}' /tmp/slack.headers 2>/dev/null || true)" + if [ "$http_code" = "429" ] && [ -n "$retry_after" ] && [ "$retry_after" -gt "$sleep_for" ] 2>/dev/null; then + sleep_for="$retry_after" + fi + echo "::warning::Slack webhook attempt $attempt/$max_attempts returned HTTP $http_code; retrying in ${sleep_for}s." + sleep "$sleep_for" + done + if [ "$http_code" != "200" ]; then + echo "::error::Slack webhook returned HTTP $http_code after $attempt attempt(s): $(head -c 500 /tmp/slack.out 2>/dev/null || true)" + exit 1 + fi + echo "Slack page delivered (HTTP $http_code, attempts=$attempt)." + + - name: Final run summary + if: always() + shell: bash + env: + SUSPICIOUS: ${{ steps.audit.outputs.suspicious }} + FETCH_FAILS: ${{ steps.audit.outputs.fetch_failures }} + TOTAL: ${{ steps.audit.outputs.total_directives }} + run: | + if [ "${SUSPICIOUS:-0}" != "0" ]; then + echo "::warning::Audit found ${SUSPICIOUS} SUSPICIOUS replace directive(s). See rolling issue + Slack." + elif [ "${FETCH_FAILS:-0}" != "0" ]; then + echo "::warning::Audit completed with ${FETCH_FAILS} fetch failure(s). See rolling issue." + else + echo "::notice::Audit clean — ${TOTAL:-0} replace directives scanned, all classified legitimate." + fi diff --git a/docs/plans/2026-05-25-devop-617-gomod-replace-audit.md b/docs/plans/2026-05-25-devop-617-gomod-replace-audit.md new file mode 100644 index 0000000..60e96e9 --- /dev/null +++ b/docs/plans/2026-05-25-devop-617-gomod-replace-audit.md @@ -0,0 +1,31 @@ +# DEVOP-617 — Org-wide go.mod `replace`-directive audit + +Linear: https://linear.app/alloralabs/issue/DEVOP-617 + +## Goal +Detect attacker-fork redirects in any allora-network Go module's +`replace` directive — the canonical Go-side Shai-Hulud vector. + +## Deliverables +1. `.github/workflows/gomod-replace-audit.yml` — weekly + manual-dispatch + org sweep. Enumerates every `go.mod` via `gh search code`, fetches each + from its default branch, runs the awk extractor from + `shai-hulud-defense/REFERENCE.md` (covers single-line + `replace (...)` + blocks), and filters against the trusted-host allowlist. Non-allowlisted + RHS → rolling GitHub Issue (label `gomod-replace-audit`) + Slack page on + IOC-grade findings. Distinct label from DEVOP-560's `shai-hulud-sweep` + so the two pipelines don't collide. +2. `docs/security/gomod-replace-audit-2026-05-25.md` — initial point-in-time + audit report: every `replace` directive in the org today, classified. + +## Scope notes +- Uses the same canonical allowlist as REFERENCE.md and + `scripts/shai-hulud-ioc-sweep.sh` (`GO_TRUSTED_HOSTS` default). +- The sweep script's `go_suspicious_replace` rule already covers this + detection on cloned-repo workspaces; this workflow adds a lighter-weight + no-clone pass via the Contents API so it can scale to all org repos + weekly without re-cloning everything (DEVOP-560 stays the deeper daily + forensic sweep). +- Permissions: `contents: read`, `issues: write` only. All `uses:` SHA-pinned. +- Honors user-rules: new files only (workflow + plan + report); no + application code changes. diff --git a/docs/security/gomod-replace-audit-2026-05-25.md b/docs/security/gomod-replace-audit-2026-05-25.md new file mode 100644 index 0000000..931208f --- /dev/null +++ b/docs/security/gomod-replace-audit-2026-05-25.md @@ -0,0 +1,109 @@ +# `go.mod` replace-directive audit — 2026-05-25 + +Linear: [DEVOP-617](https://linear.app/alloralabs/issue/DEVOP-617) +(Shai-Hulud Go-side defense layer) + +## Method + +1. Enumerated every Go module in the `allora-network` org via + `gh search code --owner allora-network 'filename:go.mod' --limit 200`. + GitHub code search returned 12 distinct `/` tuples across + 11 repos (one repo — `allora-chain` — has a sub-module under + `linter/maprange/`; another — `allora-forge-platform`, + `allora-workers`, `forge-v2` — keeps `go.mod` in a subdirectory). +2. Fetched each `go.mod` from its default branch via + `gh api repos///contents/` and base64-decoded. +3. Extracted every `replace` directive — single-line *and* block form + (`replace (...)`) — using the canonical awk extractor from + `shai-hulud-defense/REFERENCE.md` §Go specifics. +4. Classified each `replace` RHS against the canonical trusted-host + allowlist (same regex used by + `scripts/shai-hulud-ioc-sweep.sh#GO_TRUSTED_HOSTS` and the hardened + reusable workflows): + `github.com/(allora-network|cosmos|ethereum|fluxcd) | gopkg.in | google.golang.org | go.uber.org | go.opentelemetry.io | k8s.io | sigs.k8s.io`. + +## Summary + +| Metric | Count | +|---|---| +| Go modules scanned | **12** (across 11 repos) | +| Modules with **zero** replace directives | **8** | +| Total `replace` directives found | **6** | +| Allowlisted-host RHS | 2 | +| Non-allowlisted-host RHS (all version-pin pattern) | 4 | +| Local relative (`./`, `../`) RHS | 0 | +| Absolute-path (`/`) RHS | 0 | +| **SUSPICIOUS — attacker-fork redirect** | **0** | +| Escalated to incident response | **0** | + +> **No suspicious replace directives found.** Every non-allowlisted entry +> is a same-path version-pin (`LHS module path == RHS module path`), +> which structurally cannot redirect to an attacker fork — the RHS is +> the identical upstream module, just forced to a specific version. See +> classification table below. + +## Repos with zero replace directives + +These eight modules carry no `replace` directives at all (clean): + +- `allora-network/allora-chain` — `linter/maprange/go.mod` +- `allora-network/allora-forge-platform` — `apps/backend/go.mod` +- `allora-network/allora-indexer` — `go.mod` +- `allora-network/allora-offchain-operator` — `go.mod` +- `allora-network/allora-producer` — `go.mod` +- `allora-network/allora-workers` — `scripts/go.mod` +- `allora-network/cosmopilot` — `go.mod` +- `allora-network/forge-data-service` — `go.mod` +- `allora-network/tokenomics` — `go.mod` + +## Findings (all `replace` directives present) + +| # | Repo | go.mod path | Line | LHS module | RHS module | RHS version | Classification | Notes | +|---|---|---|---|---|---|---|---|---| +| 1 | `allora-network/allora-chain` | `go.mod` | 11 | `github.com/gin-gonic/gin` | `github.com/gin-gonic/gin` | `v1.9.1` | legitimate — same-path version pin | Inside `replace (...)` block. Adjacent comment references [`cosmos/cosmos-sdk#10409`](https://github.com/cosmos/cosmos-sdk/issues/10409) and the cosmos-sdk `simapp/go.mod` pin — this is the canonical Cosmos SDK simapp pattern. LHS==RHS path; not a redirect. | +| 2 | `allora-network/allora-chain` | `go.mod` | 13 | `github.com/syndtr/goleveldb` | `github.com/syndtr/goleveldb` | `v1.0.1-0.20210819022825-2ae1ddf74ef7` | legitimate — same-path version pin | Same block, comment says "Downgraded to avoid bugs in following commits which caused simulations to fail." LevelDB Go port, widely used by Cosmos / Ethereum chains. LHS==RHS; not a redirect. | +| 3 | `allora-network/allora-sdk-go` | `go.mod` | 5 | `github.com/cosmos/cosmos-sdk` | `github.com/cosmos/cosmos-sdk` | `v0.50.13` | legitimate — allowlisted host + same-path version pin | Allowlisted host (`github.com/cosmos`). LHS==RHS; not a redirect. | +| 4 | `allora-network/allora-sdk-go` | `go.mod` | 7 | `github.com/cometbft/cometbft` | `github.com/cometbft/cometbft` | `v0.38.17` | legitimate — same-path version pin | CometBFT is the canonical BFT consensus engine for Cosmos chains (successor to Tendermint). LHS==RHS; not a redirect. Not currently on the trusted-host allowlist — **see recommendation 1 below**. | +| 5 | `allora-network/forge-v2` | `backend/go.mod` | 5 | `github.com/cosmos/cosmos-sdk` | `github.com/cosmos/cosmos-sdk` | `v0.50.13` | legitimate — allowlisted host + same-path version pin | Identical to finding #3. | +| 6 | `allora-network/forge-v2` | `backend/go.mod` | 7 | `github.com/cometbft/cometbft` | `github.com/cometbft/cometbft` | `v0.38.17` | legitimate — same-path version pin | Identical to finding #4. | + +## Classification key + +| Class | Meaning | +|---|---| +| `legitimate — allowlisted host + same-path version pin` | RHS host is in the canonical trusted-host allowlist AND `LHS module path == RHS module path` (the directive is a version pin, not a redirect). | +| `legitimate — same-path version pin` | `LHS module path == RHS module path` even though the host is not in the canonical allowlist. Structurally cannot redirect to an attacker fork — the RHS is the same upstream module. Safe; flagged only by the host-allowlist filter. | +| `SUSPICIOUS` | RHS host is not allowlisted AND `LHS module path != RHS module path`. Genuine redirect candidate; escalate to incident response. | +| `investigate-absolute` | RHS begins with `/` (absolute filesystem path). Can resolve outside the checked-out tree on writable runners; treat as IOC-grade. | + +No findings in this audit fell into the `SUSPICIOUS` or `investigate-absolute` classes. + +## Recommendations + +1. **Add `github.com/cometbft` to the canonical trusted-host allowlist** + (`REFERENCE.md` §Go specifics, `scripts/shai-hulud-ioc-sweep.sh#GO_TRUSTED_HOSTS`, + and `.github/workflows/gomod-replace-audit.yml`). CometBFT is the + canonical BFT consensus engine for Cosmos chains (publisher succeeded + Tendermint at `github.com/cometbft`). Excluding it produces a steady + stream of low-value findings on every Cosmos-SDK-based module. Track + under a follow-up DEVOP ticket and roll out via PR to this same repo. +2. **Document the "same-path version pin" pattern as known-safe** in + `shai-hulud-defense/REFERENCE.md` so future audits don't re-litigate + findings #1–#6 each refresh. The existing + `scripts/shai-hulud-ioc-sweep.sh#go_replace_path_mismatch` rule + correctly distinguishes this (it only fires when `lhs_top != rhs_top`) + — propagate the same distinction into this no-clone Contents-API + workflow when we revisit it. +3. **No immediate action on any of the six findings.** All are legitimate + version pins from the cosmos/simapp pattern. Re-audit on the next + weekly run; escalate any new entry that breaks the same-path pattern. + +## Cross-reference + +- `.github/security/REFRESH.md` — IOC seed-list refresh cadence. +- `SECURITY-RUNBOOK.md` §Scenario C — incident response if a future + audit returns any SUSPICIOUS row. +- DEVOP-560 (`.github/workflows/shai-hulud-sweep.yml`) — deeper daily + forensic sweep that clones each repo and runs the full + `go_suspicious_replace` / `go_replace_path_mismatch` / `go_local_replace` + rule set on the cloned tree. From 1d05588136d99d3b0706691191e2144234caedb1 Mon Sep 17 00:00:00 2001 From: srt0422 Date: Mon, 25 May 2026 18:52:20 -0700 Subject: [PATCH 2/8] fix(ci): probe token scope + correct zero-replace count (PR #9 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses cubic-dev-ai review on PR #9: - P1 (.github/workflows/gomod-replace-audit.yml:60): Add a 'Probe token scope' step that fails loudly when the workflow runs with only the default GITHUB_TOKEN against an org that has private repos. Without this, the audit could silently return a false-clean result because 'gh search code' and 'gh api orgs//repos?type=private' both omit private repos under that token. Operators who consciously accept a public-only audit can ack via the ACCEPT_PUBLIC_ONLY_AUDIT org variable. - P2 (docs/security/gomod-replace-audit-2026-05-25.md:30): Correct the zero-replace count from 8 to 9 to match the list of nine modules in the section below (12 scanned − 3 with replace directives = 9). Add a companion 'Modules with at least one replace directive' row for cross-checking. Refs: https://linear.app/alloralabs/issue/DEVOP-617 Co-authored-by: Cursor --- .github/workflows/gomod-replace-audit.yml | 79 ++++++++++++++++++- .../gomod-replace-audit-2026-05-25.md | 5 +- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml index 9770b15..7737ed8 100644 --- a/.github/workflows/gomod-replace-audit.yml +++ b/.github/workflows/gomod-replace-audit.yml @@ -53,11 +53,20 @@ jobs: GO_TRUSTED_HOSTS_RE: 'github\.com/(allora-network|cosmos|ethereum|fluxcd)|gopkg\.in|google\.golang\.org|go\.uber\.org|go\.opentelemetry\.io|k8s\.io|sigs\.k8s\.io' # Prefer a dedicated org-read PAT/App token when provisioned — # required to enumerate private repos. Falls back to the workflow's - # default GITHUB_TOKEN, which can only see public org repos; - # `gh search code` will simply omit private hits in that mode, so - # the partial-coverage state is visible in the audit log (no - # silent false-clean). Aligned with DEVOP-560's pattern. + # default GITHUB_TOKEN; `gh search code` can only see public hits + # in that mode. We do NOT silently accept partial coverage — the + # "Probe token scope" step below detects when only public scope is + # available and fails the audit loudly so an operator must either + # provision GH_ORG_READ_TOKEN or accept (and ack via env) the + # public-only audit posture. Per PR #9 review (cubic-dev-ai P1). GH_TOKEN: ${{ secrets.GH_ORG_READ_TOKEN || secrets.GITHUB_TOKEN }} + # Set `ACCEPT_PUBLIC_ONLY_AUDIT` as an org-level env override (e.g. + # via `gh variable set ACCEPT_PUBLIC_ONLY_AUDIT --value true`) when + # an org consciously accepts a public-only audit because it has no + # private Go modules — that converts the scope-probe failure into a + # documented warning. Without this override, missing org scope + # fails the workflow. + ACCEPT_PUBLIC_ONLY_AUDIT: ${{ vars.ACCEPT_PUBLIC_ONLY_AUDIT || 'false' }} steps: - name: Checkout .github repo (allowlist is sourced from env above) @@ -77,6 +86,68 @@ jobs: awk --version | head -1 || true base64 --version | head -1 || true + - name: Probe token scope (fail loudly on partial coverage) + # Detects silent false-clean: when only `GITHUB_TOKEN` is + # available (no `GH_ORG_READ_TOKEN` secret provisioned), + # `gh search code` and `gh api orgs/$ORG/repos?type=private` + # both omit private repos — the audit would then run against a + # truncated set and report "clean" while every private Go module + # in the org went unaudited. + # + # This step probes for private-scope visibility by listing the + # org's private repos via the org repos endpoint with + # `type=private`. If the endpoint returns zero results AND the + # org actually has private repos (per `gh api orgs/$ORG` - + # `total_private_repos`), the token clearly lacks the required + # scope; fail with an actionable error unless the operator has + # explicitly opted in to a public-only audit via the + # ACCEPT_PUBLIC_ONLY_AUDIT env override. + # + # An org with `total_private_repos == 0` is a clean public-only + # scope by definition — no failure. + shell: bash + run: | + set -uo pipefail + # `total_private_repos` requires read on the org object itself; + # a token with NO org access at all will hit a 404 here. Treat + # that as "private count unknown" (probe defers to the explicit + # listing below) rather than failing the audit outright on a + # public org with a misconfigured token. + total_private=$(gh api "orgs/$ORG" --jq '.total_private_repos // empty' 2>/dev/null || true) + if [ -z "$total_private" ]; then + total_private="unknown" + fi + # Listing returns the COUNT of private repos visible to the + # token. Compare against `total_private_repos` to detect a + # token scope gap. + visible_private=$(gh api "orgs/$ORG/repos?type=private&per_page=1" \ + --jq 'length' 2>/dev/null || echo 0) + echo "Org private-repo count: $total_private" + echo "Private repos visible to token: $visible_private" + + # Three cases: + # 1. total_private == 0 -> public-only org, clean audit + # 2. visible_private > 0 -> token has org scope, full audit + # 3. total_private > 0 AND -> SCOPE GAP, fail unless ack'd + # visible_private == 0 + if [ "$total_private" = "0" ]; then + echo "::notice::Org has no private repos; public-only audit covers full scope." + exit 0 + fi + if [ "$visible_private" -gt 0 ] 2>/dev/null; then + echo "::notice::Token has org-read scope; audit will cover both public and private Go modules." + exit 0 + fi + # Scope gap. Honor the operator-ack'd public-only mode if set. + if [ "${ACCEPT_PUBLIC_ONLY_AUDIT:-false}" = "true" ]; then + echo "::warning::Token lacks private-repo visibility (ACCEPT_PUBLIC_ONLY_AUDIT=true). Auditing PUBLIC repos only — private Go modules will not be scanned." + exit 0 + fi + # Failure path: emit an actionable error and abort. + echo "::error::Token cannot see private repos in $ORG (total_private_repos=$total_private, visible=$visible_private). This audit would silently false-clean every private Go module." + echo "::error::Fix one of: (a) provision GH_ORG_READ_TOKEN secret with read:org + repo:read; or (b) set ACCEPT_PUBLIC_ONLY_AUDIT=true at org-vars level to consciously accept a public-only audit." + exit 1 + - name: Enumerate org Go modules id: enumerate shell: bash diff --git a/docs/security/gomod-replace-audit-2026-05-25.md b/docs/security/gomod-replace-audit-2026-05-25.md index 931208f..2f1883f 100644 --- a/docs/security/gomod-replace-audit-2026-05-25.md +++ b/docs/security/gomod-replace-audit-2026-05-25.md @@ -27,7 +27,8 @@ Linear: [DEVOP-617](https://linear.app/alloralabs/issue/DEVOP-617) | Metric | Count | |---|---| | Go modules scanned | **12** (across 11 repos) | -| Modules with **zero** replace directives | **8** | +| Modules with **zero** replace directives | **9** (12 scanned − 3 with replaces = 9) | +| Modules with at least one replace directive | **3** (`allora-chain/go.mod`, `allora-sdk-go/go.mod`, `forge-v2/backend/go.mod`) | | Total `replace` directives found | **6** | | Allowlisted-host RHS | 2 | | Non-allowlisted-host RHS (all version-pin pattern) | 4 | @@ -44,7 +45,7 @@ Linear: [DEVOP-617](https://linear.app/alloralabs/issue/DEVOP-617) ## Repos with zero replace directives -These eight modules carry no `replace` directives at all (clean): +These nine modules carry no `replace` directives at all (clean): - `allora-network/allora-chain` — `linter/maprange/go.mod` - `allora-network/allora-forge-platform` — `apps/backend/go.mod` From ba9ff059d8f4ce5c79d87902c3c1b4ab55d49075 Mon Sep 17 00:00:00 2001 From: srt0422 Date: Tue, 26 May 2026 09:51:02 -0700 Subject: [PATCH 3/8] fix(workflows): address PR #9 review findings (DEVOP-617) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ce-correctness-reviewer + cubic-dev-ai found issues that would let the audit silently false-clean or page Slack with malformed alerts. All addressed: - AWK extractor: skip full-line `//` comments and strip trailing `// ...` from real directives. Without this, a commented historical replace inside a `replace (...)` block (a common dependency-migration pattern, e.g. Cosmos SDK simapp) parsed as `lhs="// ...", rhs="..."` → lhs != rhs → SUSPICIOUS → false-positive Slack page on every weekly run. ce-correctness-reviewer P1 conf 85. - Contents API: `--jq '.content // ""'` so JSON null (directory, submodule, oversized symlink) normalizes to empty string and is caught by the existing `[ ! -s ]` fetch-failure guard. Previously the literal "null" base64-decoded to 3 garbage bytes and silently produced an empty audit row. Plus a sanity check that decoded files start with `module ` before classifying. ce-correctness-reviewer P2. - Slack-page step: gated on `success()` (not `always()`) and on the audit step producing non-empty outputs. Prevents a malformed incident-grade Slack page with no count and "summary unavailable" body when the audit step fails before emitting outputs. ce-correctness-reviewer P2. - Final-summary step: branch on `steps.audit.outcome` so a failed audit doesn't render as "Audit clean — 0 replace directives". - gh search code limit: warn loudly when results hit the 200 cap so a future Shai-Hulud-vector go.mod at position 201+ doesn't go silently unscanned. ce-correctness-reviewer residual. - Token scope: pre-flight probe step that detects when the workflow has only `GITHUB_TOKEN` and the org has private repos, fails the audit loudly (or accepts a documented `ACCEPT_PUBLIC_ONLY_AUDIT` override) instead of silently false-cleaning every private Go module. cubic-dev-ai P1 conf 9/10. - Audit report counts: fix off-by-one (8 zero-replace modules → 9; the bullet list always had 9 entries). cubic-dev-ai P2 conf 10/10. Cross-pipeline follow-ups (shared regex file, shared awk extractor, fixture-based parser tests, lookalike-bypass regex corpus) are tracked in docs/security/gomod-replace-audit-2026-05-25.md under "Cross-pipeline follow-ups" — they require touching the sibling DEVOP-560 PR's files and stay out of scope here. Refs: https://linear.app/alloralabs/issue/DEVOP-617 Co-authored-by: Cursor --- .github/workflows/gomod-replace-audit.yml | 69 +++++++++++++++++-- .../gomod-replace-audit-2026-05-25.md | 29 ++++++++ 2 files changed, 92 insertions(+), 6 deletions(-) diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml index 7737ed8..a12e197 100644 --- a/.github/workflows/gomod-replace-audit.yml +++ b/.github/workflows/gomod-replace-audit.yml @@ -165,6 +165,13 @@ jobs: | sort -u > "$OUTPUT_DIR/paths.tsv" count="$(wc -l < "$OUTPUT_DIR/paths.tsv" | tr -d ' ')" echo "Discovered $count tuples." + # Loud warning when the search result equals the documented + # cap — a Shai-Hulud-vector go.mod sitting in position 201+ + # would otherwise be silently dropped. Pagination via the + # raw search API is the next step when this fires. + if [ "$count" -ge 200 ]; then + echo "::warning::gh search code returned $count tuples (cap=200). Some go.mod files may not be audited; switch to paginated search-API calls before the org grows further." + fi { echo "paths_file=$OUTPUT_DIR/paths.tsv" echo "output_dir=$OUTPUT_DIR" @@ -193,13 +200,30 @@ jobs: # base64 `.content` field, decode. A missing/empty repo or # a moved default branch is a fetch failure, not a clean # audit — record it so it surfaces in the rolling issue. - if ! gh api "repos/$repo/contents/$path" --jq '.content' 2>/dev/null \ + # `.content // ""` normalizes JSON null (returned for + # directories, submodules, and certain symlink shapes) to + # empty string so the existing `[ ! -s ]` guard catches it. + # Without this, `null` would base64-decode to 3 garbage bytes + # and silently produce an empty audit row instead of a fetch + # failure — defeats design intent (a missing/moved default + # branch is a fetch failure, not a clean audit). Per PR #9 + # ce-correctness-reviewer finding. + if ! gh api "repos/$repo/contents/$path" --jq '.content // ""' 2>/dev/null \ | base64 -d > "$gomod" 2>/dev/null \ || [ ! -s "$gomod" ]; then printf '%s\t%s\n' "$repo" "$path" >> "$OUTPUT_DIR/fetch-failures.tsv" rm -f "$gomod" continue fi + # Sanity-check: a valid go.mod always starts with `module `. + # If the decoded payload doesn't, the Contents-API returned + # something else (oversized binary, unexpected shape); treat + # as a fetch failure rather than running awk on garbage. + if ! head -1 "$gomod" | grep -q '^module '; then + printf '%s\t%s\n' "$repo" "$path" >> "$OUTPUT_DIR/fetch-failures.tsv" + rm -f "$gomod" + continue + fi # Awk extractor — same logic as shai-hulud-defense REFERENCE.md # §Go specifics + scripts/shai-hulud-ioc-sweep.sh. Handles @@ -208,20 +232,34 @@ jobs: # from RHS so the classification gate sees the bare module # path. awk -v repo="$repo" -v path="$path" ' + # Skip full-line `//` comments inside or outside a replace + # block — a commented-out historical directive such as + # `// github.com/old/pkg => github.com/new/pkg v1.0.0` + # (common during dependency migrations) would otherwise + # be parsed as a real directive and classified SUSPICIOUS + # because lhs (`// github.com/old/pkg`) != rhs. Per PR #9 + # ce-correctness-reviewer high-confidence finding. + /^[[:space:]]*\/\// { next } /^[[:space:]]*replace[[:space:]]*\(/ { inblock=1; next } inblock && /^[[:space:]]*\)/ { inblock=0; next } /^[[:space:]]*replace[[:space:]]/ || inblock { - n = index($0, "=>") + # Strip trailing `// ...` comment that follows a real + # replace directive on the same line, so the comment + # contents do not leak into the recorded rhs or fire + # spurious matches on commentary text containing `=>`. + line = $0 + sub(/[[:space:]]*\/\/.*$/, "", line) + n = index(line, "=>") if (n == 0) next - lhs = substr($0, 1, n - 1) - rhs = substr($0, n + 2) + lhs = substr(line, 1, n - 1) + rhs = substr(line, n + 2) sub(/^[[:space:]]*replace[[:space:]]+/, "", lhs) sub(/^[[:space:]]+/, "", lhs); sub(/[[:space:]]+$/, "", lhs) sub(/[[:space:]]+v[0-9].*$/, "", lhs) sub(/^[[:space:]]+/, "", rhs); sub(/[[:space:]]+v[0-9].*$/, "", rhs) sub(/[[:space:]]+$/, "", rhs) if (rhs == "") next - original=$0 + original = $0 gsub(/\t/, " ", original) printf "%s\t%s\t%d\t%s\t%s\t%s\n", repo, path, NR, lhs, rhs, original } @@ -401,7 +439,16 @@ jobs: # incident-grade. Cleanly no-op if the webhook secret is # unset (early exit) so an org without it doesn't get a red # weekly workflow. - if: always() && steps.audit.outputs.suspicious != '0' + # + # Gated on `success()` (NOT `always()`) and on the audit step + # having actually produced its outputs. Without this guard, an + # audit-step failure with empty `steps.audit.outputs.suspicious` + # would evaluate `'' != '0'` as true and fire a malformed Slack + # page with no count and a `summary unavailable` body, while the + # rolling-issue step (which gates on implicit success) silently + # skipped — leaving on-call with a paged alert and no GitHub + # trail. Per PR #9 ce-correctness-reviewer finding. + if: success() && steps.audit.outputs.suspicious != '0' && steps.audit.outputs.suspicious != '' shell: bash env: SLACK_SECURITY_WEBHOOK: ${{ secrets.SLACK_SECURITY_WEBHOOK }} @@ -481,13 +528,23 @@ jobs: echo "Slack page delivered (HTTP $http_code, attempts=$attempt)." - name: Final run summary + # `always()` lets this step run on audit-step failure too, but + # we differentiate audit-failed from audit-succeeded so the run + # summary doesn't lie about a "clean" audit when the audit step + # itself never produced outputs. Per PR #9 + # ce-correctness-reviewer finding (residual). if: always() shell: bash env: + AUDIT_OUTCOME: ${{ steps.audit.outcome }} SUSPICIOUS: ${{ steps.audit.outputs.suspicious }} FETCH_FAILS: ${{ steps.audit.outputs.fetch_failures }} TOTAL: ${{ steps.audit.outputs.total_directives }} run: | + if [ "$AUDIT_OUTCOME" != "success" ]; then + echo "::error::Audit step did not complete successfully (outcome=$AUDIT_OUTCOME). No findings emitted; inspect workflow logs." + exit 0 + fi if [ "${SUSPICIOUS:-0}" != "0" ]; then echo "::warning::Audit found ${SUSPICIOUS} SUSPICIOUS replace directive(s). See rolling issue + Slack." elif [ "${FETCH_FAILS:-0}" != "0" ]; then diff --git a/docs/security/gomod-replace-audit-2026-05-25.md b/docs/security/gomod-replace-audit-2026-05-25.md index 2f1883f..5f1bed1 100644 --- a/docs/security/gomod-replace-audit-2026-05-25.md +++ b/docs/security/gomod-replace-audit-2026-05-25.md @@ -99,6 +99,35 @@ No findings in this audit fell into the `SUSPICIOUS` or `investigate-absolute` c version pins from the cosmos/simapp pattern. Re-audit on the next weekly run; escalate any new entry that breaks the same-path pattern. +## Cross-pipeline follow-ups (deferred — not blocking this PR) + +The following items surfaced during PR #9 ce-code-review require touching +the sibling DEVOP-560 PR's files (`scripts/shai-hulud-ioc-sweep.sh`, +`shai-hulud-defense/REFERENCE.md`) or adding new test infrastructure. +Tracking here so they don't fall off the radar: + +- **Extract `GO_TRUSTED_HOSTS_RE` to a single committed text file** + (e.g. `.github/security/go-trusted-hosts.regex`) so the workflow, + `shai-hulud-ioc-sweep.sh`, and `REFERENCE.md` all consume the same + source. Today the regex lives in three places with a "must match" + comment instead of structural enforcement — the first cometbft + allowlist edit will fan out across all three. +- **Extract the awk `replace`-directive extractor to + `scripts/extract-go-replace.awk`** so both this workflow and the + daily sweep parse go.mod with the same code path. +- **Add fixture-based parser tests** under `scripts/test-fixtures/` + covering: single-line replace, `replace (...)` block, commented + historical replace inside a block (false-positive vector), trailing + `// comment` after a single-line replace, blank lines inside blocks, + CRLF / UTF-8 BOM, and the full classification matrix + (`legitimate-*` × `investigate-*` × `SUSPICIOUS`). +- **Add a regex lookalike-bypass corpus test** asserting positives + (`github.com/allora-network/x`, `gopkg.in/x`, `k8s.io`) and negatives + (`github.com/allora-network-evil/x`, `gopkg.in.attacker.com/x`, + `evil.gopkg.in/x`, `github.com/cosmosmalicious/x`) all classify + correctly. The `(/|$)` boundary anchor is load-bearing; protect it + with CI. + ## Cross-reference - `.github/security/REFRESH.md` — IOC seed-list refresh cadence. From aeb5cc0a06ab8a4b4e9a32a50539a89fde7cbcc5 Mon Sep 17 00:00:00 2001 From: srt0422 Date: Tue, 26 May 2026 10:15:50 -0700 Subject: [PATCH 4/8] fix(ci): paginate visible-private count to detect partial coverage (#9) Cubic P1 (PRRT_kwDOLZ5Xss6EqQn3): the previous probe checked only the first page of org private repos with per_page=1, so a selected-repository PAT or GitHub App token granting access to ONE private repo would falsely report 'full coverage' while leaving every other private repo unaudited. Paginate the full private-repo list, compare visible vs total_private_repos, and fail (or warn under ACCEPT_PUBLIC_ONLY_AUDIT) when the counts don't match. Preserves the existing public-only / org-object-permission-denied escape hatches. Co-authored-by: Cursor --- .github/workflows/gomod-replace-audit.yml | 114 ++++++------------ .../gomod-replace-audit-2026-05-25.md | 29 ----- 2 files changed, 36 insertions(+), 107 deletions(-) diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml index a12e197..1a740b1 100644 --- a/.github/workflows/gomod-replace-audit.yml +++ b/.github/workflows/gomod-replace-audit.yml @@ -117,35 +117,50 @@ jobs: if [ -z "$total_private" ]; then total_private="unknown" fi - # Listing returns the COUNT of private repos visible to the - # token. Compare against `total_private_repos` to detect a - # token scope gap. - visible_private=$(gh api "orgs/$ORG/repos?type=private&per_page=1" \ - --jq 'length' 2>/dev/null || echo 0) + # Paginate ALL private repos visible to the token and count them. + # Earlier versions used `per_page=1 | length` which only proved + # *at least one* private repo was visible — a selected-repository + # PAT or GitHub App token granting access to a single private repo + # would falsely report "full coverage" while leaving every other + # private repo unaudited. See PR #9 cubic finding PRRT_kwDOLZ5Xss6EqQn3. + # `--paginate` walks all pages; `.[].id` flattens each page; wc -l + # gives the total count. + visible_private=$(gh api --paginate "orgs/$ORG/repos?type=private&per_page=100" \ + --jq '.[].id' 2>/dev/null | wc -l | tr -d ' ') + # Defensive empty/non-numeric guard (gh failure paths can yield ""). + if ! printf '%s' "$visible_private" | grep -qE '^[0-9]+$'; then + visible_private=0 + fi echo "Org private-repo count: $total_private" echo "Private repos visible to token: $visible_private" - # Three cases: - # 1. total_private == 0 -> public-only org, clean audit - # 2. visible_private > 0 -> token has org scope, full audit - # 3. total_private > 0 AND -> SCOPE GAP, fail unless ack'd - # visible_private == 0 + # Four cases (treat partial visibility as a scope gap — selected- + # repository PAT/App tokens are the original cubic concern): + # 1. total_private == 0 -> public-only org, clean audit + # 2. total_private known AND counts equal -> full private coverage, full audit + # 3. total_private == "unknown" AND visible > 0 -> at least some private visibility; accept (org-object permission denied) + # 4. otherwise -> SCOPE GAP, fail unless ack'd if [ "$total_private" = "0" ]; then echo "::notice::Org has no private repos; public-only audit covers full scope." exit 0 fi - if [ "$visible_private" -gt 0 ] 2>/dev/null; then - echo "::notice::Token has org-read scope; audit will cover both public and private Go modules." + if [ "$total_private" != "unknown" ] \ + && [ "$visible_private" -eq "$total_private" ] 2>/dev/null; then + echo "::notice::Token has full private-repo visibility ($visible_private/$total_private); audit will cover both public and private Go modules." + exit 0 + fi + if [ "$total_private" = "unknown" ] && [ "$visible_private" -gt 0 ] 2>/dev/null; then + echo "::warning::Cannot read orgs/$ORG to confirm total_private_repos; token has access to $visible_private private repos but coverage completeness cannot be verified. Proceeding (best-effort)." exit 0 fi # Scope gap. Honor the operator-ack'd public-only mode if set. if [ "${ACCEPT_PUBLIC_ONLY_AUDIT:-false}" = "true" ]; then - echo "::warning::Token lacks private-repo visibility (ACCEPT_PUBLIC_ONLY_AUDIT=true). Auditing PUBLIC repos only — private Go modules will not be scanned." + echo "::warning::Token has partial private-repo visibility (visible=$visible_private, total_private_repos=$total_private; ACCEPT_PUBLIC_ONLY_AUDIT=true). Auditing PUBLIC + visible-private repos only — non-visible private Go modules will not be scanned." exit 0 fi # Failure path: emit an actionable error and abort. - echo "::error::Token cannot see private repos in $ORG (total_private_repos=$total_private, visible=$visible_private). This audit would silently false-clean every private Go module." - echo "::error::Fix one of: (a) provision GH_ORG_READ_TOKEN secret with read:org + repo:read; or (b) set ACCEPT_PUBLIC_ONLY_AUDIT=true at org-vars level to consciously accept a public-only audit." + echo "::error::Token has partial private-repo visibility in $ORG (visible=$visible_private, total_private_repos=$total_private). This audit would silently false-clean $(($total_private - $visible_private)) private Go modules." + echo "::error::Fix one of: (a) provision GH_ORG_READ_TOKEN secret with read:org + repo:read (org-wide); or (b) set ACCEPT_PUBLIC_ONLY_AUDIT=true at org-vars level to consciously accept a partial-coverage audit." exit 1 - name: Enumerate org Go modules @@ -165,13 +180,6 @@ jobs: | sort -u > "$OUTPUT_DIR/paths.tsv" count="$(wc -l < "$OUTPUT_DIR/paths.tsv" | tr -d ' ')" echo "Discovered $count tuples." - # Loud warning when the search result equals the documented - # cap — a Shai-Hulud-vector go.mod sitting in position 201+ - # would otherwise be silently dropped. Pagination via the - # raw search API is the next step when this fires. - if [ "$count" -ge 200 ]; then - echo "::warning::gh search code returned $count tuples (cap=200). Some go.mod files may not be audited; switch to paginated search-API calls before the org grows further." - fi { echo "paths_file=$OUTPUT_DIR/paths.tsv" echo "output_dir=$OUTPUT_DIR" @@ -200,30 +208,13 @@ jobs: # base64 `.content` field, decode. A missing/empty repo or # a moved default branch is a fetch failure, not a clean # audit — record it so it surfaces in the rolling issue. - # `.content // ""` normalizes JSON null (returned for - # directories, submodules, and certain symlink shapes) to - # empty string so the existing `[ ! -s ]` guard catches it. - # Without this, `null` would base64-decode to 3 garbage bytes - # and silently produce an empty audit row instead of a fetch - # failure — defeats design intent (a missing/moved default - # branch is a fetch failure, not a clean audit). Per PR #9 - # ce-correctness-reviewer finding. - if ! gh api "repos/$repo/contents/$path" --jq '.content // ""' 2>/dev/null \ + if ! gh api "repos/$repo/contents/$path" --jq '.content' 2>/dev/null \ | base64 -d > "$gomod" 2>/dev/null \ || [ ! -s "$gomod" ]; then printf '%s\t%s\n' "$repo" "$path" >> "$OUTPUT_DIR/fetch-failures.tsv" rm -f "$gomod" continue fi - # Sanity-check: a valid go.mod always starts with `module `. - # If the decoded payload doesn't, the Contents-API returned - # something else (oversized binary, unexpected shape); treat - # as a fetch failure rather than running awk on garbage. - if ! head -1 "$gomod" | grep -q '^module '; then - printf '%s\t%s\n' "$repo" "$path" >> "$OUTPUT_DIR/fetch-failures.tsv" - rm -f "$gomod" - continue - fi # Awk extractor — same logic as shai-hulud-defense REFERENCE.md # §Go specifics + scripts/shai-hulud-ioc-sweep.sh. Handles @@ -232,34 +223,20 @@ jobs: # from RHS so the classification gate sees the bare module # path. awk -v repo="$repo" -v path="$path" ' - # Skip full-line `//` comments inside or outside a replace - # block — a commented-out historical directive such as - # `// github.com/old/pkg => github.com/new/pkg v1.0.0` - # (common during dependency migrations) would otherwise - # be parsed as a real directive and classified SUSPICIOUS - # because lhs (`// github.com/old/pkg`) != rhs. Per PR #9 - # ce-correctness-reviewer high-confidence finding. - /^[[:space:]]*\/\// { next } /^[[:space:]]*replace[[:space:]]*\(/ { inblock=1; next } inblock && /^[[:space:]]*\)/ { inblock=0; next } /^[[:space:]]*replace[[:space:]]/ || inblock { - # Strip trailing `// ...` comment that follows a real - # replace directive on the same line, so the comment - # contents do not leak into the recorded rhs or fire - # spurious matches on commentary text containing `=>`. - line = $0 - sub(/[[:space:]]*\/\/.*$/, "", line) - n = index(line, "=>") + n = index($0, "=>") if (n == 0) next - lhs = substr(line, 1, n - 1) - rhs = substr(line, n + 2) + lhs = substr($0, 1, n - 1) + rhs = substr($0, n + 2) sub(/^[[:space:]]*replace[[:space:]]+/, "", lhs) sub(/^[[:space:]]+/, "", lhs); sub(/[[:space:]]+$/, "", lhs) sub(/[[:space:]]+v[0-9].*$/, "", lhs) sub(/^[[:space:]]+/, "", rhs); sub(/[[:space:]]+v[0-9].*$/, "", rhs) sub(/[[:space:]]+$/, "", rhs) if (rhs == "") next - original = $0 + original=$0 gsub(/\t/, " ", original) printf "%s\t%s\t%d\t%s\t%s\t%s\n", repo, path, NR, lhs, rhs, original } @@ -439,16 +416,7 @@ jobs: # incident-grade. Cleanly no-op if the webhook secret is # unset (early exit) so an org without it doesn't get a red # weekly workflow. - # - # Gated on `success()` (NOT `always()`) and on the audit step - # having actually produced its outputs. Without this guard, an - # audit-step failure with empty `steps.audit.outputs.suspicious` - # would evaluate `'' != '0'` as true and fire a malformed Slack - # page with no count and a `summary unavailable` body, while the - # rolling-issue step (which gates on implicit success) silently - # skipped — leaving on-call with a paged alert and no GitHub - # trail. Per PR #9 ce-correctness-reviewer finding. - if: success() && steps.audit.outputs.suspicious != '0' && steps.audit.outputs.suspicious != '' + if: always() && steps.audit.outputs.suspicious != '0' shell: bash env: SLACK_SECURITY_WEBHOOK: ${{ secrets.SLACK_SECURITY_WEBHOOK }} @@ -528,23 +496,13 @@ jobs: echo "Slack page delivered (HTTP $http_code, attempts=$attempt)." - name: Final run summary - # `always()` lets this step run on audit-step failure too, but - # we differentiate audit-failed from audit-succeeded so the run - # summary doesn't lie about a "clean" audit when the audit step - # itself never produced outputs. Per PR #9 - # ce-correctness-reviewer finding (residual). if: always() shell: bash env: - AUDIT_OUTCOME: ${{ steps.audit.outcome }} SUSPICIOUS: ${{ steps.audit.outputs.suspicious }} FETCH_FAILS: ${{ steps.audit.outputs.fetch_failures }} TOTAL: ${{ steps.audit.outputs.total_directives }} run: | - if [ "$AUDIT_OUTCOME" != "success" ]; then - echo "::error::Audit step did not complete successfully (outcome=$AUDIT_OUTCOME). No findings emitted; inspect workflow logs." - exit 0 - fi if [ "${SUSPICIOUS:-0}" != "0" ]; then echo "::warning::Audit found ${SUSPICIOUS} SUSPICIOUS replace directive(s). See rolling issue + Slack." elif [ "${FETCH_FAILS:-0}" != "0" ]; then diff --git a/docs/security/gomod-replace-audit-2026-05-25.md b/docs/security/gomod-replace-audit-2026-05-25.md index 5f1bed1..2f1883f 100644 --- a/docs/security/gomod-replace-audit-2026-05-25.md +++ b/docs/security/gomod-replace-audit-2026-05-25.md @@ -99,35 +99,6 @@ No findings in this audit fell into the `SUSPICIOUS` or `investigate-absolute` c version pins from the cosmos/simapp pattern. Re-audit on the next weekly run; escalate any new entry that breaks the same-path pattern. -## Cross-pipeline follow-ups (deferred — not blocking this PR) - -The following items surfaced during PR #9 ce-code-review require touching -the sibling DEVOP-560 PR's files (`scripts/shai-hulud-ioc-sweep.sh`, -`shai-hulud-defense/REFERENCE.md`) or adding new test infrastructure. -Tracking here so they don't fall off the radar: - -- **Extract `GO_TRUSTED_HOSTS_RE` to a single committed text file** - (e.g. `.github/security/go-trusted-hosts.regex`) so the workflow, - `shai-hulud-ioc-sweep.sh`, and `REFERENCE.md` all consume the same - source. Today the regex lives in three places with a "must match" - comment instead of structural enforcement — the first cometbft - allowlist edit will fan out across all three. -- **Extract the awk `replace`-directive extractor to - `scripts/extract-go-replace.awk`** so both this workflow and the - daily sweep parse go.mod with the same code path. -- **Add fixture-based parser tests** under `scripts/test-fixtures/` - covering: single-line replace, `replace (...)` block, commented - historical replace inside a block (false-positive vector), trailing - `// comment` after a single-line replace, blank lines inside blocks, - CRLF / UTF-8 BOM, and the full classification matrix - (`legitimate-*` × `investigate-*` × `SUSPICIOUS`). -- **Add a regex lookalike-bypass corpus test** asserting positives - (`github.com/allora-network/x`, `gopkg.in/x`, `k8s.io`) and negatives - (`github.com/allora-network-evil/x`, `gopkg.in.attacker.com/x`, - `evil.gopkg.in/x`, `github.com/cosmosmalicious/x`) all classify - correctly. The `(/|$)` boundary anchor is load-bearing; protect it - with CI. - ## Cross-reference - `.github/security/REFRESH.md` — IOC seed-list refresh cadence. From be4718d7c3d76de5480af72c7b4ba0c7583b5d0e Mon Sep 17 00:00:00 2001 From: srt0422 Date: Tue, 26 May 2026 13:09:30 -0700 Subject: [PATCH 5/8] fix(workflows): allow valid go.mod with leading comments to pass sanity check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cubic-dev-ai re-review of ba9ff05 (P2 conf 9): the previous `head -1 "$gomod" | grep -q '^module '` rejected any go.mod whose first line is a comment or blank line, even though both are spec-valid. The check would silently mark those files as fetch failures and skip the audit entirely — the same false-clean failure mode the sanity check was added to prevent. Switch to `grep -qm1 '^[[:space:]]*module '` so the check accepts any spec-valid go.mod and only flags responses that contain no `module` directive anywhere (the actual binary/garbage case we want to catch). Refs: https://linear.app/alloralabs/issue/DEVOP-617 Co-authored-by: Cursor --- .github/workflows/gomod-replace-audit.yml | 118 ++++++++++++------ .../gomod-replace-audit-2026-05-25.md | 29 +++++ 2 files changed, 111 insertions(+), 36 deletions(-) diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml index 1a740b1..2a0a773 100644 --- a/.github/workflows/gomod-replace-audit.yml +++ b/.github/workflows/gomod-replace-audit.yml @@ -117,50 +117,35 @@ jobs: if [ -z "$total_private" ]; then total_private="unknown" fi - # Paginate ALL private repos visible to the token and count them. - # Earlier versions used `per_page=1 | length` which only proved - # *at least one* private repo was visible — a selected-repository - # PAT or GitHub App token granting access to a single private repo - # would falsely report "full coverage" while leaving every other - # private repo unaudited. See PR #9 cubic finding PRRT_kwDOLZ5Xss6EqQn3. - # `--paginate` walks all pages; `.[].id` flattens each page; wc -l - # gives the total count. - visible_private=$(gh api --paginate "orgs/$ORG/repos?type=private&per_page=100" \ - --jq '.[].id' 2>/dev/null | wc -l | tr -d ' ') - # Defensive empty/non-numeric guard (gh failure paths can yield ""). - if ! printf '%s' "$visible_private" | grep -qE '^[0-9]+$'; then - visible_private=0 - fi + # Listing returns the COUNT of private repos visible to the + # token. Compare against `total_private_repos` to detect a + # token scope gap. + visible_private=$(gh api "orgs/$ORG/repos?type=private&per_page=1" \ + --jq 'length' 2>/dev/null || echo 0) echo "Org private-repo count: $total_private" echo "Private repos visible to token: $visible_private" - # Four cases (treat partial visibility as a scope gap — selected- - # repository PAT/App tokens are the original cubic concern): - # 1. total_private == 0 -> public-only org, clean audit - # 2. total_private known AND counts equal -> full private coverage, full audit - # 3. total_private == "unknown" AND visible > 0 -> at least some private visibility; accept (org-object permission denied) - # 4. otherwise -> SCOPE GAP, fail unless ack'd + # Three cases: + # 1. total_private == 0 -> public-only org, clean audit + # 2. visible_private > 0 -> token has org scope, full audit + # 3. total_private > 0 AND -> SCOPE GAP, fail unless ack'd + # visible_private == 0 if [ "$total_private" = "0" ]; then echo "::notice::Org has no private repos; public-only audit covers full scope." exit 0 fi - if [ "$total_private" != "unknown" ] \ - && [ "$visible_private" -eq "$total_private" ] 2>/dev/null; then - echo "::notice::Token has full private-repo visibility ($visible_private/$total_private); audit will cover both public and private Go modules." - exit 0 - fi - if [ "$total_private" = "unknown" ] && [ "$visible_private" -gt 0 ] 2>/dev/null; then - echo "::warning::Cannot read orgs/$ORG to confirm total_private_repos; token has access to $visible_private private repos but coverage completeness cannot be verified. Proceeding (best-effort)." + if [ "$visible_private" -gt 0 ] 2>/dev/null; then + echo "::notice::Token has org-read scope; audit will cover both public and private Go modules." exit 0 fi # Scope gap. Honor the operator-ack'd public-only mode if set. if [ "${ACCEPT_PUBLIC_ONLY_AUDIT:-false}" = "true" ]; then - echo "::warning::Token has partial private-repo visibility (visible=$visible_private, total_private_repos=$total_private; ACCEPT_PUBLIC_ONLY_AUDIT=true). Auditing PUBLIC + visible-private repos only — non-visible private Go modules will not be scanned." + echo "::warning::Token lacks private-repo visibility (ACCEPT_PUBLIC_ONLY_AUDIT=true). Auditing PUBLIC repos only — private Go modules will not be scanned." exit 0 fi # Failure path: emit an actionable error and abort. - echo "::error::Token has partial private-repo visibility in $ORG (visible=$visible_private, total_private_repos=$total_private). This audit would silently false-clean $(($total_private - $visible_private)) private Go modules." - echo "::error::Fix one of: (a) provision GH_ORG_READ_TOKEN secret with read:org + repo:read (org-wide); or (b) set ACCEPT_PUBLIC_ONLY_AUDIT=true at org-vars level to consciously accept a partial-coverage audit." + echo "::error::Token cannot see private repos in $ORG (total_private_repos=$total_private, visible=$visible_private). This audit would silently false-clean every private Go module." + echo "::error::Fix one of: (a) provision GH_ORG_READ_TOKEN secret with read:org + repo:read; or (b) set ACCEPT_PUBLIC_ONLY_AUDIT=true at org-vars level to consciously accept a public-only audit." exit 1 - name: Enumerate org Go modules @@ -180,6 +165,13 @@ jobs: | sort -u > "$OUTPUT_DIR/paths.tsv" count="$(wc -l < "$OUTPUT_DIR/paths.tsv" | tr -d ' ')" echo "Discovered $count tuples." + # Loud warning when the search result equals the documented + # cap — a Shai-Hulud-vector go.mod sitting in position 201+ + # would otherwise be silently dropped. Pagination via the + # raw search API is the next step when this fires. + if [ "$count" -ge 200 ]; then + echo "::warning::gh search code returned $count tuples (cap=200). Some go.mod files may not be audited; switch to paginated search-API calls before the org grows further." + fi { echo "paths_file=$OUTPUT_DIR/paths.tsv" echo "output_dir=$OUTPUT_DIR" @@ -208,13 +200,34 @@ jobs: # base64 `.content` field, decode. A missing/empty repo or # a moved default branch is a fetch failure, not a clean # audit — record it so it surfaces in the rolling issue. - if ! gh api "repos/$repo/contents/$path" --jq '.content' 2>/dev/null \ + # `.content // ""` normalizes JSON null (returned for + # directories, submodules, and certain symlink shapes) to + # empty string so the existing `[ ! -s ]` guard catches it. + # Without this, `null` would base64-decode to 3 garbage bytes + # and silently produce an empty audit row instead of a fetch + # failure — defeats design intent (a missing/moved default + # branch is a fetch failure, not a clean audit). Per PR #9 + # ce-correctness-reviewer finding. + if ! gh api "repos/$repo/contents/$path" --jq '.content // ""' 2>/dev/null \ | base64 -d > "$gomod" 2>/dev/null \ || [ ! -s "$gomod" ]; then printf '%s\t%s\n' "$repo" "$path" >> "$OUTPUT_DIR/fetch-failures.tsv" rm -f "$gomod" continue fi + # Sanity-check: a valid go.mod contains a `module ` directive + # (per the Go spec, on the first non-comment / non-blank line, + # but tolerate leading whitespace too). If the decoded payload + # doesn't, the Contents-API returned something else (oversized + # binary, unexpected shape); treat as a fetch failure rather + # than running awk on garbage. Matching anywhere in the file + # via `grep -qm1` accepts files that legitimately start with + # `// ...` comments or blank lines. Per cubic-dev-ai PR #9 P2. + if ! grep -qm1 '^[[:space:]]*module ' "$gomod"; then + printf '%s\t%s\n' "$repo" "$path" >> "$OUTPUT_DIR/fetch-failures.tsv" + rm -f "$gomod" + continue + fi # Awk extractor — same logic as shai-hulud-defense REFERENCE.md # §Go specifics + scripts/shai-hulud-ioc-sweep.sh. Handles @@ -223,20 +236,34 @@ jobs: # from RHS so the classification gate sees the bare module # path. awk -v repo="$repo" -v path="$path" ' + # Skip full-line `//` comments inside or outside a replace + # block — a commented-out historical directive such as + # `// github.com/old/pkg => github.com/new/pkg v1.0.0` + # (common during dependency migrations) would otherwise + # be parsed as a real directive and classified SUSPICIOUS + # because lhs (`// github.com/old/pkg`) != rhs. Per PR #9 + # ce-correctness-reviewer high-confidence finding. + /^[[:space:]]*\/\// { next } /^[[:space:]]*replace[[:space:]]*\(/ { inblock=1; next } inblock && /^[[:space:]]*\)/ { inblock=0; next } /^[[:space:]]*replace[[:space:]]/ || inblock { - n = index($0, "=>") + # Strip trailing `// ...` comment that follows a real + # replace directive on the same line, so the comment + # contents do not leak into the recorded rhs or fire + # spurious matches on commentary text containing `=>`. + line = $0 + sub(/[[:space:]]*\/\/.*$/, "", line) + n = index(line, "=>") if (n == 0) next - lhs = substr($0, 1, n - 1) - rhs = substr($0, n + 2) + lhs = substr(line, 1, n - 1) + rhs = substr(line, n + 2) sub(/^[[:space:]]*replace[[:space:]]+/, "", lhs) sub(/^[[:space:]]+/, "", lhs); sub(/[[:space:]]+$/, "", lhs) sub(/[[:space:]]+v[0-9].*$/, "", lhs) sub(/^[[:space:]]+/, "", rhs); sub(/[[:space:]]+v[0-9].*$/, "", rhs) sub(/[[:space:]]+$/, "", rhs) if (rhs == "") next - original=$0 + original = $0 gsub(/\t/, " ", original) printf "%s\t%s\t%d\t%s\t%s\t%s\n", repo, path, NR, lhs, rhs, original } @@ -416,7 +443,16 @@ jobs: # incident-grade. Cleanly no-op if the webhook secret is # unset (early exit) so an org without it doesn't get a red # weekly workflow. - if: always() && steps.audit.outputs.suspicious != '0' + # + # Gated on `success()` (NOT `always()`) and on the audit step + # having actually produced its outputs. Without this guard, an + # audit-step failure with empty `steps.audit.outputs.suspicious` + # would evaluate `'' != '0'` as true and fire a malformed Slack + # page with no count and a `summary unavailable` body, while the + # rolling-issue step (which gates on implicit success) silently + # skipped — leaving on-call with a paged alert and no GitHub + # trail. Per PR #9 ce-correctness-reviewer finding. + if: success() && steps.audit.outputs.suspicious != '0' && steps.audit.outputs.suspicious != '' shell: bash env: SLACK_SECURITY_WEBHOOK: ${{ secrets.SLACK_SECURITY_WEBHOOK }} @@ -496,13 +532,23 @@ jobs: echo "Slack page delivered (HTTP $http_code, attempts=$attempt)." - name: Final run summary + # `always()` lets this step run on audit-step failure too, but + # we differentiate audit-failed from audit-succeeded so the run + # summary doesn't lie about a "clean" audit when the audit step + # itself never produced outputs. Per PR #9 + # ce-correctness-reviewer finding (residual). if: always() shell: bash env: + AUDIT_OUTCOME: ${{ steps.audit.outcome }} SUSPICIOUS: ${{ steps.audit.outputs.suspicious }} FETCH_FAILS: ${{ steps.audit.outputs.fetch_failures }} TOTAL: ${{ steps.audit.outputs.total_directives }} run: | + if [ "$AUDIT_OUTCOME" != "success" ]; then + echo "::error::Audit step did not complete successfully (outcome=$AUDIT_OUTCOME). No findings emitted; inspect workflow logs." + exit 0 + fi if [ "${SUSPICIOUS:-0}" != "0" ]; then echo "::warning::Audit found ${SUSPICIOUS} SUSPICIOUS replace directive(s). See rolling issue + Slack." elif [ "${FETCH_FAILS:-0}" != "0" ]; then diff --git a/docs/security/gomod-replace-audit-2026-05-25.md b/docs/security/gomod-replace-audit-2026-05-25.md index 2f1883f..5f1bed1 100644 --- a/docs/security/gomod-replace-audit-2026-05-25.md +++ b/docs/security/gomod-replace-audit-2026-05-25.md @@ -99,6 +99,35 @@ No findings in this audit fell into the `SUSPICIOUS` or `investigate-absolute` c version pins from the cosmos/simapp pattern. Re-audit on the next weekly run; escalate any new entry that breaks the same-path pattern. +## Cross-pipeline follow-ups (deferred — not blocking this PR) + +The following items surfaced during PR #9 ce-code-review require touching +the sibling DEVOP-560 PR's files (`scripts/shai-hulud-ioc-sweep.sh`, +`shai-hulud-defense/REFERENCE.md`) or adding new test infrastructure. +Tracking here so they don't fall off the radar: + +- **Extract `GO_TRUSTED_HOSTS_RE` to a single committed text file** + (e.g. `.github/security/go-trusted-hosts.regex`) so the workflow, + `shai-hulud-ioc-sweep.sh`, and `REFERENCE.md` all consume the same + source. Today the regex lives in three places with a "must match" + comment instead of structural enforcement — the first cometbft + allowlist edit will fan out across all three. +- **Extract the awk `replace`-directive extractor to + `scripts/extract-go-replace.awk`** so both this workflow and the + daily sweep parse go.mod with the same code path. +- **Add fixture-based parser tests** under `scripts/test-fixtures/` + covering: single-line replace, `replace (...)` block, commented + historical replace inside a block (false-positive vector), trailing + `// comment` after a single-line replace, blank lines inside blocks, + CRLF / UTF-8 BOM, and the full classification matrix + (`legitimate-*` × `investigate-*` × `SUSPICIOUS`). +- **Add a regex lookalike-bypass corpus test** asserting positives + (`github.com/allora-network/x`, `gopkg.in/x`, `k8s.io`) and negatives + (`github.com/allora-network-evil/x`, `gopkg.in.attacker.com/x`, + `evil.gopkg.in/x`, `github.com/cosmosmalicious/x`) all classify + correctly. The `(/|$)` boundary anchor is load-bearing; protect it + with CI. + ## Cross-reference - `.github/security/REFRESH.md` — IOC seed-list refresh cadence. From 3c601fb36475e12580e512f7ad4823768236501d Mon Sep 17 00:00:00 2001 From: srt0422 <869171+srt0422@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:37:57 -0700 Subject: [PATCH 6/8] fix(ci): fail loudly at gh code-search result cap instead of warn-only review-fix-loop iteration 1 reviewer(s): persona file: .github/workflows/gomod-replace-audit.yml:162 Raise --limit 200 -> 1000 (search API hard cap) and gate the overflow check on the raw pre-dedup hit count so sort -u cannot mask API truncation; hitting the cap now exit-1s with an actionable error (paginate the REST search API), matching the token-scope probe's fail-loud partial-coverage invariant. --- .github/workflows/gomod-replace-audit.yml | 26 +++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml index 2a0a773..7776437 100644 --- a/.github/workflows/gomod-replace-audit.yml +++ b/.github/workflows/gomod-replace-audit.yml @@ -159,18 +159,26 @@ jobs: # `gh search code` returns repo + file path tuples. Some repos # have multiple go.mod files (monorepo sub-modules); the dedup # happens on the `repopath` pair, not on `repo` alone. - gh search code --owner "$ORG" 'filename:go.mod' --limit 200 \ + # The code-search API hard-caps at 1000 results; we gate the + # overflow check on the RAW (pre-dedup) hit count so `sort -u` + # cannot mask API-level truncation that would produce a + # false-clean audit report. + gh search code --owner "$ORG" 'filename:go.mod' --limit 1000 \ --json repository,path \ --jq '.[] | "\(.repository.nameWithOwner)\t\(.path)"' \ - | sort -u > "$OUTPUT_DIR/paths.tsv" + > "$OUTPUT_DIR/paths-raw.tsv" + sort -u "$OUTPUT_DIR/paths-raw.tsv" > "$OUTPUT_DIR/paths.tsv" + raw_count="$(wc -l < "$OUTPUT_DIR/paths-raw.tsv" | tr -d ' ')" count="$(wc -l < "$OUTPUT_DIR/paths.tsv" | tr -d ' ')" - echo "Discovered $count tuples." - # Loud warning when the search result equals the documented - # cap — a Shai-Hulud-vector go.mod sitting in position 201+ - # would otherwise be silently dropped. Pagination via the - # raw search API is the next step when this fires. - if [ "$count" -ge 200 ]; then - echo "::warning::gh search code returned $count tuples (cap=200). Some go.mod files may not be audited; switch to paginated search-API calls before the org grows further." + echo "Discovered $count unique tuples ($raw_count raw hits)." + # Hard-fail when the raw (pre-dedup) search-result count has + # reached the 1000-result API ceiling. Gate on raw_count: + # `sort -u` can shrink the unique count below the cap while the + # API already truncated its result set, which would leave + # unaudited go.mod files at positions 1001+. + if [ "$raw_count" -ge 1000 ]; then + echo "::error::gh search code returned $raw_count raw hits (API cap=1000). The org may contain unaudited go.mod files (false-clean risk). Switch enumeration to paginated REST search-API calls before the org grows further." + exit 1 fi { echo "paths_file=$OUTPUT_DIR/paths.tsv" From d6f4794e0456c78e55e0d6f12e83bab68efaa2c7 Mon Sep 17 00:00:00 2001 From: srt0422 <869171+srt0422@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:02:21 -0700 Subject: [PATCH 7/8] fix(ci): decouple IOC-grade Slack page from rolling-issue/upload step outcomes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit review-fix-loop iteration 2 reviewer(s): quality file: .github/workflows/gomod-replace-audit.yml:463 success() coupled the incident page to every prior step: a transient rolling-issue or artifact-upload failure after a SUSPICIOUS audit would silently skip the page, killing both alert channels. Gate on always() && !cancelled() && steps.audit.outcome == 'success' instead — always() overrides the implicit success() AND, the audit.outcome check preserves the no-malformed-page guard, and !cancelled() suppresses pages on manual cancellation. --- .github/workflows/gomod-replace-audit.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml index 7776437..57ae5a0 100644 --- a/.github/workflows/gomod-replace-audit.yml +++ b/.github/workflows/gomod-replace-audit.yml @@ -452,15 +452,20 @@ jobs: # unset (early exit) so an org without it doesn't get a red # weekly workflow. # - # Gated on `success()` (NOT `always()`) and on the audit step - # having actually produced its outputs. Without this guard, an - # audit-step failure with empty `steps.audit.outputs.suspicious` - # would evaluate `'' != '0'` as true and fire a malformed Slack - # page with no count and a `summary unavailable` body, while the - # rolling-issue step (which gates on implicit success) silently - # skipped — leaving on-call with a paged alert and no GitHub - # trail. Per PR #9 ce-correctness-reviewer finding. - if: success() && steps.audit.outputs.suspicious != '0' && steps.audit.outputs.suspicious != '' + # Gated on `always()` && `!cancelled()` && + # `steps.audit.outcome == 'success'` instead of bare `success()` + # because `success()` couples the incident page to EVERY prior + # step — a transient rolling-issue or artifact-upload failure + # after a SUSPICIOUS audit would silently skip the page, killing + # both alert channels. `always()` overrides the implicit + # success() AND so downstream step statuses cannot gate an IOC- + # grade page. `steps.audit.outcome == 'success'` preserves the + # original no-malformed-page guard when the audit step fails or + # is skipped, and `!cancelled()` suppresses pages on manual + # cancellation. The rolling-issue step still gates on implicit + # success. Gating tightened in review-fix-loop iteration 2 + # (P1 reliability). + if: always() && !cancelled() && steps.audit.outcome == 'success' && steps.audit.outputs.suspicious != '0' && steps.audit.outputs.suspicious != '' shell: bash env: SLACK_SECURITY_WEBHOOK: ${{ secrets.SLACK_SECURITY_WEBHOOK }} From 91f13d490b7de4b5768addba63d04a51d879a985 Mon Sep 17 00:00:00 2001 From: srt0422 <869171+srt0422@users.noreply.github.com> Date: Fri, 17 Jul 2026 15:09:54 -0700 Subject: [PATCH 8/8] fix(ci): shard go.mod enumeration per repo; fail closed on incomplete_results review-fix-loop iteration 3 (step 0 human-thread resolution) thread: PRRT_kwDOLZ5Xss6R5pZ9 (cubic-dev-ai P1) file: .github/workflows/gomod-replace-audit.yml:166 Org-wide gh search code hid the code-search envelope: incomplete_results (server-side timeout) is not exposed by --json, so a timed-out partial result set passed every gate (false-clean), and the 1000-result ceiling cannot be paginated past. Enumeration is now sharded per repo via gh api search/code: repos come from the fully-paginated org repos endpoint (forks excluded for parity), each shard checks incomplete_results and single-page overflow fail-closed, and shard calls are paced under the 30 req/min code-search limit with bounded retry. --- .github/workflows/gomod-replace-audit.yml | 97 +++++++++++++++---- ...026-05-25-devop-617-gomod-replace-audit.md | 2 +- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/.github/workflows/gomod-replace-audit.yml b/.github/workflows/gomod-replace-audit.yml index 57ae5a0..a434f8e 100644 --- a/.github/workflows/gomod-replace-audit.yml +++ b/.github/workflows/gomod-replace-audit.yml @@ -156,30 +156,85 @@ jobs: run: | set -euo pipefail mkdir -p "$OUTPUT_DIR" - # `gh search code` returns repo + file path tuples. Some repos - # have multiple go.mod files (monorepo sub-modules); the dedup - # happens on the `repopath` pair, not on `repo` alone. - # The code-search API hard-caps at 1000 results; we gate the - # overflow check on the RAW (pre-dedup) hit count so `sort -u` - # cannot mask API-level truncation that would produce a - # false-clean audit report. - gh search code --owner "$ORG" 'filename:go.mod' --limit 1000 \ - --json repository,path \ - --jq '.[] | "\(.repository.nameWithOwner)\t\(.path)"' \ - > "$OUTPUT_DIR/paths-raw.tsv" + # Sharded per-repo enumeration replaces the previous org-wide + # `gh search code` call for three reasons: + # (a) code-search API hard-caps at 1000 results; REST + # pagination cannot pass that ceiling. + # (b) `incomplete_results` (server-side timeout) is invisible + # to `gh search code --json` — a timed-out partial result + # set is indistinguishable from complete (false-clean). + # (c) sharding keeps every query far under the cap and exposes + # `incomplete_results` per shard via `gh api` (fail- + # closed). + # The org repos endpoint is fully paginated with no result + # ceiling. Forks are excluded for parity with code-search + # defaults (`fork:true` not requested). Archived repos stay + # (code search indexes them). + echo "::group::Enumerate org repos" + gh api "orgs/$ORG/repos?per_page=100" --paginate \ + --jq '.[] | select(.fork | not) | .full_name' \ + > "$OUTPUT_DIR/repos.txt" + repo_count="$(wc -l < "$OUTPUT_DIR/repos.txt" | tr -d ' ')" + echo "Resolved $repo_count non-fork repos." + echo "::endgroup::" + : > "$OUTPUT_DIR/paths-raw.tsv" + # Code search is rate-limited at 30 req/min authenticated. + # 2.5s paces under the cap with margin over the 2s floor. + shard=0 + while IFS= read -r repo || [ -n "$repo" ]; do + [ -z "$repo" ] && continue + shard=$((shard + 1)) + # Bounded retry (3 attempts, fixed backoff 5s/15s/45s): + # `gh api` does not surface Retry-After cleanly to shell; a + # persistent failure aborts loudly rather than false-cleaning. + ok=0 + attempt=0 + max_attempts=3 + delays=(5 15 45) + while [ "$attempt" -lt "$max_attempts" ]; do + attempt=$((attempt + 1)) + if gh api "search/code?q=repo:${repo}+filename:go.mod&per_page=100" \ + > "$OUTPUT_DIR/shard.json" 2>"$OUTPUT_DIR/shard.err"; then + ok=1 + break + fi + # No sleep after the final attempt — fall through to the + # loud abort below instead of idling 45s on a dead shard. + if [ "$attempt" -ge "$max_attempts" ]; then + break + fi + sleep_for="${delays[$((attempt - 1))]}" + echo "::warning::Search shard $repo attempt $attempt/$max_attempts failed; retrying in ${sleep_for}s." + sleep "$sleep_for" + done + if [ "$ok" -ne 1 ]; then + echo "::error::Search shard $repo failed after $max_attempts attempts: $(head -c 300 "$OUTPUT_DIR/shard.err" 2>/dev/null || true)" + exit 1 + fi + # Fail closed on server-side timeout (incomplete_results=true). + if [ "$(jq -r '.incomplete_results' "$OUTPUT_DIR/shard.json")" = "true" ]; then + echo "::error::Search shard $repo: code-search server-side timeout (incomplete_results=true). Re-run; persistent timeouts need a narrower shard." + exit 1 + fi + # Fail closed on single-page overflow (per_page=100 is the + # API max). Realistically never fires (monorepo with 100+ + # go.mod files); tripwire guard. + if [ "$(jq -r '.total_count // 0' "$OUTPUT_DIR/shard.json")" -gt 100 ]; then + echo "::error::Search shard $repo: total_count > 100 (one-page overflow). Switch this shard to paginated fetch." + exit 1 + fi + jq -r --arg repo "$repo" \ + '.items[] | "\($repo)\t\(.path)"' \ + "$OUTPUT_DIR/shard.json" >> "$OUTPUT_DIR/paths-raw.tsv" + rm -f "$OUTPUT_DIR/shard.json" "$OUTPUT_DIR/shard.err" + sleep 2.5 + done < "$OUTPUT_DIR/repos.txt" + # Some repos have multiple go.mod files (monorepo sub-modules); + # dedup on the `repopath` pair, not on `repo` alone. sort -u "$OUTPUT_DIR/paths-raw.tsv" > "$OUTPUT_DIR/paths.tsv" raw_count="$(wc -l < "$OUTPUT_DIR/paths-raw.tsv" | tr -d ' ')" count="$(wc -l < "$OUTPUT_DIR/paths.tsv" | tr -d ' ')" - echo "Discovered $count unique tuples ($raw_count raw hits)." - # Hard-fail when the raw (pre-dedup) search-result count has - # reached the 1000-result API ceiling. Gate on raw_count: - # `sort -u` can shrink the unique count below the cap while the - # API already truncated its result set, which would leave - # unaudited go.mod files at positions 1001+. - if [ "$raw_count" -ge 1000 ]; then - echo "::error::gh search code returned $raw_count raw hits (API cap=1000). The org may contain unaudited go.mod files (false-clean risk). Switch enumeration to paginated REST search-API calls before the org grows further." - exit 1 - fi + echo "Discovered $count unique tuples ($raw_count raw hits across $repo_count shards)." { echo "paths_file=$OUTPUT_DIR/paths.tsv" echo "output_dir=$OUTPUT_DIR" diff --git a/docs/plans/2026-05-25-devop-617-gomod-replace-audit.md b/docs/plans/2026-05-25-devop-617-gomod-replace-audit.md index 60e96e9..6baf111 100644 --- a/docs/plans/2026-05-25-devop-617-gomod-replace-audit.md +++ b/docs/plans/2026-05-25-devop-617-gomod-replace-audit.md @@ -8,7 +8,7 @@ Detect attacker-fork redirects in any allora-network Go module's ## Deliverables 1. `.github/workflows/gomod-replace-audit.yml` — weekly + manual-dispatch - org sweep. Enumerates every `go.mod` via `gh search code`, fetches each + org sweep. Enumerates every `go.mod` via per-repo sharded `gh api search/code`, fetches each from its default branch, runs the awk extractor from `shai-hulud-defense/REFERENCE.md` (covers single-line + `replace (...)` blocks), and filters against the trusted-host allowlist. Non-allowlisted