From dd720d5266cf63525638112ee931f60256a9c1ae Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:53:02 -0700 Subject: [PATCH] =?UTF-8?q?chore(ci):=20four=20per-PR=20runner=20reduction?= =?UTF-8?q?s=20=E2=80=94=20fold=20dependency=20review=20into=20changes,=20?= =?UTF-8?q?make=20selfhost=20push-only,=20one-runner=20same-repo=20preview?= =?UTF-8?q?s,=20merge=20the=20calibration=20advisories?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the validate-tests unsharding (#8514), attacking the remaining per-PR runner fan-out at the source (observed 44-deep queue with 3 running): 1. The standalone `security` job folds into `changes` as a PR-only dependency-review step -- one fewer runner slot on EVERY PR at zero wall-clock cost (the API lockfile diff takes seconds). `validate`'s needs drops it; a dep failure still reddens `validate` via `changes`. 2. selfhost.yml drops its pull_request trigger (~99 x 5.2min/week -- migrations/** made nearly every backend PR pay a docker build+boot). ci.yml still validates migrations/schema/pg suites pre-merge; the boot smoke now catches a break minutes after merge, and self-host users only consume tagged releases, never main. 3. Same-repo UI PRs build AND deploy their preview in ONE job: the deploy half moves to a shared .github/actions/deploy-ui-preview composite (setup-workspace precedent), ui-preview.yml gains the inline build-deploy job, and ui-preview-deploy.yml becomes fork-only (its job skips runner-free for same-repo builds via head_repository comparison). The fork trust boundary is untouched: fork code still builds with zero secrets and deploys only through the trusted workflow_run half. Bonus: the deploy job now has a trusted default-branch checkout, retiring the hand-synced hardcoded Node version. 4. backtest-logic-check.yml + counterfactual-replay-check.yml merge into calibration-advisory.yml -- identical scaffolding, overlapping paths (ai-review.ts fired BOTH), so one job runs both halves for one npm ci and one runner slot. The replay half keeps its budget spend-guard and now self-gates on the canonical prompt actually differing (replacing its ai-review.ts-only path filter). Fail-open posture preserved verbatim on every advisory step. Net per-PR floor drops 5 -> 4 slots, UI PRs 7 -> 5, migration PRs lose the 5-minute docker job. Pinned tests updated (ci-skip-draft-prs, workflow-runner-labels); skill reference.md rows updated. --- .../contributing-to-loopover/reference.md | 6 +- .github/actions/deploy-ui-preview/action.yml | 175 ++++++++++ .github/workflows/backtest-logic-check.yml | 156 --------- .github/workflows/calibration-advisory.yml | 301 ++++++++++++++++++ .github/workflows/ci.yml | 56 ++-- .../workflows/counterfactual-replay-check.yml | 195 ------------ .github/workflows/selfhost.yml | 39 +-- .github/workflows/ui-preview-deploy.yml | 262 ++++----------- .github/workflows/ui-preview.yml | 143 +++++++-- scripts/backtest-logic-check.ts | 2 +- test/unit/ci-skip-draft-prs.test.ts | 17 +- test/unit/workflow-runner-labels.test.ts | 14 +- 12 files changed, 724 insertions(+), 642 deletions(-) create mode 100644 .github/actions/deploy-ui-preview/action.yml delete mode 100644 .github/workflows/backtest-logic-check.yml create mode 100644 .github/workflows/calibration-advisory.yml delete mode 100644 .github/workflows/counterfactual-replay-check.yml diff --git a/.claude/skills/contributing-to-loopover/reference.md b/.claude/skills/contributing-to-loopover/reference.md index 14cfdb0124..ba30822c89 100644 --- a/.claude/skills/contributing-to-loopover/reference.md +++ b/.claude/skills/contributing-to-loopover/reference.md @@ -15,7 +15,7 @@ for maintainer approval (CI shows unverified → the engine **holds**, never clo ## 1. Every CI check → local command → what fails it The **required** status checks on `main` are **`validate`** (it aggregates `changes, lint, test, -workers, mcp, ui, security`; a path-skipped job counts as success) and **`Superagent Security Scan`** +workers, mcp, ui`; a path-skipped job counts as success) and **`Superagent Security Scan`** (a separate third-party GitHub App check, not part of this repo's own workflow files — confirmed via `gh api repos/JSONbored/loopover/branches/main/protection/required_status_checks`). **Codecov** posts `codecov/patch` (the real coverage gate) and `codecov/project` (informational) independently. The @@ -91,9 +91,9 @@ regression-guards the script itself against reverting to `vite preview`. | ui → extension lint | `eslint` (VS Code + miner extensions) | `npm run extension:lint && npm run miner-extension:lint` | extension ESLint error (same `push \|\| ui==true` trigger as the `ui →` rows) | | ui → extension typecheck | `tsc --noEmit` (extensions) | `npm run extension:typecheck && npm run miner-extension:typecheck` | extension type error (same `push \|\| ui==true` trigger) | -| security (PR only) | dependency-review (moderate+) | `npm audit --audit-level=moderate` | a **newly added** dep has a moderate+ advisory | +| changes → dependency review (PR only) | dependency-review (moderate+; a step inside the `changes` job since 2026-07-24, not a separate job) | `npm audit --audit-level=moderate` | a **newly added** dep has a moderate+ advisory | -**One command for *almost* everything except `security`:** `npm run test:ci`. There is **no** CodeQL/Analyze +**One command for *almost* everything except the dependency review:** `npm run test:ci`. There is **no** CodeQL/Analyze workflow in this repo. There is **no** root-level Prettier gate — Prettier is enforced only inside `ui:lint` (so it only bites `apps/loopover-ui/**`). diff --git a/.github/actions/deploy-ui-preview/action.yml b/.github/actions/deploy-ui-preview/action.yml new file mode 100644 index 0000000000..5572b104db --- /dev/null +++ b/.github/actions/deploy-ui-preview/action.yml @@ -0,0 +1,175 @@ +name: Deploy UI preview +description: >- + The trusted deploy half of the per-PR preview pipeline, shared by its two call sites so they can + never drift (the setup-workspace precedent): ui-preview.yml deploys same-repo PRs inline in the + build job, and ui-preview-deploy.yml deploys fork PRs from the workflow_run trust boundary. It + validates a built dist bundle, writes the preview Wrangler config HERE (never taken from the PR -- + a fork cannot control bindings, routes, or vars), uploads a 0%-traffic preview version (a + workers.dev URL; wrangler only uploads the bundle, it never executes it), and records the GitHub + Deployment + success status that Reviewbot reads for the "after" screenshot. Callers must have + Node available (actions/setup-node) before invoking, and keep their own failure()-gated step that + records a `failure` deployment_status (that step is caller-specific: it must also catch failures + of the caller's OWN earlier steps, which this action cannot see). +inputs: + pr-number: + description: The pull request number this preview belongs to (Reviewbot re-reviews it on success). + required: true + head-sha: + description: The PR head SHA the deployment record points at. Pass only GitHub-set values, never fork-supplied data. + required: true + dist-dir: + description: Path to the built UI bundle (server/ + client/), relative to the workspace. + required: true + cloudflare-api-token: + description: Cloudflare API token with "Workers Scripts:Edit" on the account. + required: true + cloudflare-account-id: + description: The Cloudflare account id that owns loopover-ui. + required: true + github-token: + description: Token with deployments:write, used to record the Deployment for Reviewbot. + required: true +outputs: + preview_url: + description: The workers.dev preview URL the upload produced. + value: ${{ steps.upload.outputs.preview_url }} +runs: + using: composite + steps: + - name: Install trusted Wrangler + shell: bash + run: npm install --global wrangler@4.95.0 + + # The bundle may have been produced by an UNTRUSTED build (fork code; harmless for the trusted + # same-repo call site, where the same checks still catch build anomalies). Validate it before + # handing it to wrangler: reject symlinks (a path-traversal / exfil vector when the bundle is + # processed), require the expected SSR build structure, and allowlist file extensions so a + # malicious build can't smuggle scripts/binaries/unexpected paths into the deploy. + - name: Validate dist bundle + shell: bash + run: | + set -euo pipefail + cd "${{ inputs.dist-dir }}" + # 1) No symlinks anywhere in the bundle. + symlinks="$(find . -type l)" + if [ -n "$symlinks" ]; then + echo "::error::Bundle contains symlinks — refusing to deploy:" + printf '%s\n' "$symlinks" + exit 1 + fi + # 2) Required SSR build structure (server worker entry + client assets dir). + test -f server/index.mjs || { echo "::error::bundle missing server/index.mjs"; exit 1; } + test -d client || { echo "::error::bundle missing client/ assets dir"; exit 1; } + # 3) Allowlist file extensions — fail on anything that isn't a normal web/build output (blocks + # smuggled scripts/binaries). A few extensionless CF asset files are explicitly permitted. + # `zip` covers the served downloads (e.g. /downloads/loopover-extension.zip) — a passive + # static asset wrangler only uploads (never executes), so allowing it doesn't run fork code. + unexpected="$(find . -regextype posix-extended -type f \ + -not -iregex '.*\.(mjs|js|cjs|map|json|css|html?|txt|svg|png|jpe?g|gif|webp|avif|ico|bmp|woff2?|ttf|otf|eot|wasm|xml|webmanifest|md|csv|zip|wgsl|glb|gltf)$' \ + -not -name '_headers' -not -name '_redirects' -not -name '_routes.json' -not -name '.assetsignore')" + if [ -n "$unexpected" ]; then + echo "::error::Bundle contains unexpected file types — refusing to deploy:" + printf '%s\n' "$unexpected" + exit 1 + fi + echo "Bundle validated: no symlinks, expected SSR structure, allowlisted file types only." + + # The Wrangler config is written HERE (trusted) — never taken from the PR — so a fork cannot + # control bindings, routes, or vars. It points at the validated bundle. It deliberately OMITS the + # production custom-domain route: `wrangler versions upload` creates a 0%-traffic preview version + # (a workers.dev URL) and applies no routes, so the route is unused here — and omitting it + # guarantees that even a future switch to `wrangler deploy` could never point fork-built code at + # the production domain. Do not add a `routes` block to this preview config. + - name: Write trusted preview Wrangler config + shell: bash + run: | + cat > "${{ inputs.dist-dir }}/server/wrangler.preview.json" <<'JSON' + { + "compatibility_date": "2026-05-28", + "name": "loopover-ui", + "workers_dev": true, + "preview_urls": true, + "compatibility_flags": ["nodejs_compat"], + "placement": { + "mode": "smart" + }, + "observability": { + "enabled": true, + "logs": { + "enabled": true, + "head_sampling_rate": 1 + }, + "traces": { + "enabled": true, + "head_sampling_rate": 1 + } + }, + "vars": { + "VITE_LOOPOVER_API_ORIGIN": "https://api.loopover.ai" + }, + "main": "index.mjs", + "assets": { + "binding": "ASSETS", + "directory": "../client" + }, + "no_bundle": true, + "rules": [ + { + "type": "ESModule", + "globs": ["**/*.mjs", "**/*.js"] + } + ] + } + JSON + + - name: Upload preview version + id: upload + shell: bash + env: + CLOUDFLARE_API_TOKEN: ${{ inputs.cloudflare-api-token }} + CLOUDFLARE_ACCOUNT_ID: ${{ inputs.cloudflare-account-id }} + run: | + set -o pipefail + out=$(wrangler versions upload --config "${{ inputs.dist-dir }}/server/wrangler.preview.json" 2>&1 | tee /dev/stderr) + # Take a workers.dev URL that belongs to the loopover-ui worker, so any other URL in the logs + # (or a changed output format) can't be recorded as the preview by mistake. Tolerant of the + # version-alias prefix (`-loopover-ui..workers.dev`). + url=$(printf '%s\n' "$out" | grep -oiE 'https://[a-z0-9.-]+\.workers\.dev' | grep -i 'loopover-ui' | head -n1) + if [ -z "$url" ]; then + echo "::error::Could not parse an expected loopover-ui preview URL from wrangler output" + exit 1 + fi + echo "preview_url=$url" >> "$GITHUB_OUTPUT" + echo "Preview: $url" + + - name: Record deployment for Reviewbot + if: ${{ steps.upload.outputs.preview_url != '' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + github-token: ${{ inputs.github-token }} + script: | + const url = ${{ toJSON(steps.upload.outputs.preview_url) }}; + const sha = ${{ toJSON(inputs.head-sha) }}; + const prNumber = Number(${{ toJSON(inputs.pr-number) }}); + const deployment = await github.rest.repos.createDeployment({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: sha, + environment: `preview/pr-${prNumber}`, + auto_merge: false, + required_contexts: [], + transient_environment: true, + description: "LoopOver UI preview", + // Reviewbot reads `pr` here to re-review this exact PR once the preview is live. + payload: JSON.stringify({ pr: prNumber, head_sha: sha }), + }); + await github.rest.repos.createDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: deployment.data.id, + state: "success", + environment: `preview/pr-${prNumber}`, + environment_url: url, + description: "Preview ready", + }); + core.notice(`Preview deployment recorded for PR #${prNumber}: ${url}`); diff --git a/.github/workflows/backtest-logic-check.yml b/.github/workflows/backtest-logic-check.yml deleted file mode 100644 index ad3d3a7047..0000000000 --- a/.github/workflows/backtest-logic-check.yml +++ /dev/null @@ -1,156 +0,0 @@ -# Logic/regex-change backtest (#8139, epic #8082). When a PR touches the watched detection-logic paths, -# this job replays linked_issue_scope_mismatch's recorded raw-context history (#8129/#8130 + the #8139 -# model-response capture) through BOTH the PR's own head checkout and its base checkout — actually executing -# the two versions of the detection code, which is why this lives in CI and not in ORB's live Worker (the -# Worker holds credentials and must never execute PR-supplied logic; a CI checkout running a PR's own code -# is the exact trust boundary validate-code/validate-tests already use for every PR). Deliberately a -# separate workflow, not a ci.yml job: PRs that don't touch these paths pay nothing, and ones that do aren't -# slowed — this runs fully parallel to (and finishes long before) the ~11-minute test shards. -# Advisory only: never a required check, never blocks merge (#8105). It posts its OWN clearly-labeled PR -# comment, separate from ORB's unified review comment — see #8139's Boundaries for why. -name: backtest-logic - -on: - pull_request: - # Explicit list because the default (opened/synchronize/reopened) omits ready_for_review -- once the - # draft guard below skips draft PRs, marking a PR ready must itself trigger a real run (#6670). - # Mirrors selfhost.yml's pull_request.types comment/list exactly. - types: [opened, synchronize, reopened, ready_for_review] - # Exactly the paths whose changes can alter linked_issue_scope_mismatch-adjacent detection logic — - # see #8139's Design section; keep this list in sync with the issue's own spec. - paths: - - "src/rules/**" - - "src/review/content-lane/**" - - "src/settings/agent-actions.ts" - - "src/services/ai-review.ts" - - "src/services/linked-issue-satisfaction.ts" - -# Least privilege: the backtest only reads the repo; pull-requests: write is for its own advisory comment. -permissions: - contents: read - pull-requests: write - -concurrency: - # pull_request-only workflow, so one ref-scoped group suffices (no push/github.sha split like ci.yml - # needs): a newer push cancels the superseded run — its comment would be overwritten anyway. - group: backtest-logic-${{ github.ref }} - cancel-in-progress: true - -jobs: - backtest: - name: logic backtest (advisory) - # Skip draft PRs (#6670, anti-abuse — mirrors selfhost.yml's guard). Fork PRs are excluded at the job - # level rather than per-step: GitHub withholds repo secrets from fork-originated pull_request runs, so - # the D1 corpus read below is impossible there and the whole job (npm ci included) would be waste — - # the fork-notice job below is this workflow's half of ci.yml's paired fork==true/!=true convention. - if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork != true }} - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - # The PR's base commit, checked out INSIDE the head workspace: the dynamically imported base modules - # resolve bare npm specifiers by walking up from their own directory into the head checkout's - # node_modules, so one `npm ci` serves both sides of the comparison. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - ref: ${{ github.event.pull_request.base.sha }} - path: .backtest-base - persist-credentials: false - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version-file: .nvmrc - cache: "npm" - - - name: Install deps - run: npm ci --ignore-scripts - - # The scripts import @loopover/engine, which resolves to its dist/ build output. - - name: Build engine package - run: npx turbo run build --filter=@loopover/engine - - # Every step from here down FAILS OPEN (notice + green, never a red check): the review engine - # auto-closes a contributor PR on ANY failed check, required or not — so an advisory job that can go - # red on an infra problem (an under-scoped CLOUDFLARE_API_TOKEN, a D1 outage, a comment-post hiccup) - # would let OUR plumbing close an innocent contributor's PR. "Never blocks merge" (#8105) has to hold - # against this job's own failures, not just its verdicts. - # - # Read-only corpus export (#8084's CLI, reused as-is — no new D1 read code). The wrangler secrets are - # available here because the fork guard above already excluded fork-originated runs. - - name: Export corpus from D1 - id: corpus - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - run: | - if npx tsx scripts/backtest-corpus-export.ts --rule-id linked_issue_scope_mismatch --output backtest-corpus.json --remote; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - echo "::notice::Corpus export from D1 failed (missing or under-scoped CLOUDFLARE_API_TOKEN — it needs D1 read/write on the loopover database). Logic backtest skipped; advisory only, never fails the PR." - fi - - - name: Run logic backtest - id: backtest - if: ${{ steps.corpus.outputs.available == 'true' }} - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - if npx tsx scripts/backtest-logic-check.ts \ - --rule-id linked_issue_scope_mismatch \ - --corpus backtest-corpus.json \ - --head-root . \ - --base-root .backtest-base \ - --output backtest-comment.md \ - --head-sha "$HEAD_SHA" \ - --base-sha "$BASE_SHA" \ - --persist --remote --db loopover \ - --repo "$GITHUB_REPOSITORY" \ - --pr "$PR_NUMBER"; then - echo "ready=true" >> "$GITHUB_OUTPUT" - else - echo "ready=false" >> "$GITHUB_OUTPUT" - echo "::notice::Logic backtest run failed — no comparison produced. Advisory only, never fails the PR." - fi - - # Update-in-place keyed on the comment marker so a re-run edits the existing comment instead of - # stacking a new one per push. - - name: Post or update the PR comment - if: ${{ steps.backtest.outputs.ready == 'true' }} - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - post_comment() { - marker="" - # --paginate runs the --jq filter once per page, so pin to the first emitted id. - comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1) - if [ -n "$comment_id" ]; then - gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@backtest-comment.md - else - gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@backtest-comment.md - fi - } - if ! post_comment; then - echo "::notice::PR comment post failed — backtest result computed and persisted but not posted. Advisory only, never fails the PR." - fi - - # The fork half of ci.yml's paired fork==true/!=true convention: fork-originated pull_request runs get no - # repo secrets, so the D1-backed backtest cannot run — say so visibly instead of failing or going silent. - # Advisory only either way; a skipped backtest never blocks anything (#8105). - fork-notice: - name: logic backtest (skipped for fork PRs) - if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork == true }} - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Explain the skip - run: echo "::notice::Fork PR — repo secrets are withheld, so the D1-backed logic backtest is skipped. Advisory only; nothing blocks." diff --git a/.github/workflows/calibration-advisory.yml b/.github/workflows/calibration-advisory.yml new file mode 100644 index 0000000000..ba9b6cc06a --- /dev/null +++ b/.github/workflows/calibration-advisory.yml @@ -0,0 +1,301 @@ +# Calibration advisory checks — the logic/regex-change backtest (#8139, epic #8082) and the +# counterfactual prompt replay (#8222, sub-epic #8218, epic #8211 track C), merged into ONE workflow +# 2026-07-24: they shared identical scaffolding (dual base/head checkout, npm ci, engine build, D1 +# corpus export, fail-open posture, fork-notice pairing) and overlapping path filters +# (src/services/ai-review.ts triggered BOTH — two runners doing the same npm ci on every judge-prompt +# PR), so one job runs both halves for one setup cost and one runner slot. +# +# BACKTEST half: replays linked_issue_scope_mismatch's recorded raw-context history (#8129/#8130 + +# the #8139 model-response capture) through BOTH the PR's head checkout and its base checkout — +# actually executing the two versions of the detection code, which is why this lives in CI and not in +# ORB's live Worker (the Worker holds credentials and must never execute PR-supplied logic; a CI +# checkout running a PR's own code is the exact trust boundary validate-code/validate-tests already +# use for every PR). +# +# REPLAY half: replays the CANONICAL judge prompt from both checkouts (scripts/print-review-prompt.ts) +# against the recorded raw-context fixture corpus and posts the Pareto-floored comparison. It +# self-gates twice: on the COUNTERFACTUAL_REPLAY_BUDGET spend guard (#8222's requirement — unset, it +# posts a visible skip notice and spends nothing), and on the canonical prompt text actually differing +# between base and head (byte-identical ⇒ nothing to replay), so no extra path filter is needed for +# runs that only touched the backtest's watched paths. +# +# Advisory only (#8105): never a required check, never blocks merge, and every step after setup FAILS +# OPEN (notice + green) — the review engine auto-closes contributor PRs on ANY red check, so this +# workflow's own plumbing must never redden. Each half posts its OWN clearly-labeled PR comment, +# update-in-place keyed on its marker. +name: calibration-advisory + +on: + pull_request: + # Explicit list because the default (opened/synchronize/reopened) omits ready_for_review -- once the + # draft guard below skips draft PRs, marking a PR ready must itself trigger a real run (#6670). + types: [opened, synchronize, reopened, ready_for_review] + # The union of the two halves' watched surfaces. Backtest: exactly the paths whose changes can alter + # linked_issue_scope_mismatch-adjacent detection logic (#8139's Design section). Replay: the judge- + # prompt surface (REVIEW_PROMPT_VERSION + buildSystemPrompt/parseModelReview), which is + # src/services/ai-review.ts — already in the backtest's list, so the union IS the backtest's list. + paths: + - "src/rules/**" + - "src/review/content-lane/**" + - "src/settings/agent-actions.ts" + - "src/services/ai-review.ts" + - "src/services/linked-issue-satisfaction.ts" + +# Least privilege: the checks only read the repo; pull-requests: write is for their advisory comments. +permissions: + contents: read + pull-requests: write + +concurrency: + # pull_request-only workflow, so one ref-scoped group suffices (no push/github.sha split like ci.yml + # needs): a newer push cancels the superseded run — its comments would be overwritten anyway. + group: calibration-advisory-${{ github.ref }} + cancel-in-progress: true + +jobs: + advisory: + name: calibration advisory (backtest + replay) + # Skip draft PRs (#6670, anti-abuse). Fork PRs are excluded at the job level rather than per-step: + # GitHub withholds repo secrets from fork-originated pull_request runs, so the D1 corpus reads below + # are impossible there and the whole job (npm ci included) would be waste — the fork-notice job below + # is this workflow's half of ci.yml's paired fork==true/!=true convention. + if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork != true }} + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + # The PR's base commit, checked out INSIDE the head workspace: the dynamically imported base modules + # resolve bare npm specifiers by walking up from their own directory into the head checkout's + # node_modules, so one `npm ci` serves both sides of every comparison (#8139). + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .advisory-base + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version-file: .nvmrc + cache: "npm" + + - name: Install deps + run: npm ci --ignore-scripts + + # Both halves' scripts import @loopover/engine, which resolves to its dist/ build output. + - name: Build engine package + run: npx turbo run build --filter=@loopover/engine + + # --------------------------------------------------------------------------------------------- + # Every step from here down FAILS OPEN (notice + green, never a red check): the review engine + # auto-closes a contributor PR on ANY failed check, required or not — so an advisory job that can + # go red on an infra problem (an under-scoped CLOUDFLARE_API_TOKEN, a D1 outage, a comment-post + # hiccup) would let OUR plumbing close an innocent contributor's PR. "Never blocks merge" (#8105) + # has to hold against this job's own failures, not just its verdicts. + # --------------------------------------------------------------------------------------------- + + # ============================ BACKTEST HALF (#8139) ============================ + # Read-only corpus export (#8084's CLI, reused as-is — no new D1 read code). The wrangler secrets + # are available here because the fork guard above already excluded fork-originated runs. + - name: Export backtest corpus from D1 + id: corpus + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + if npx tsx scripts/backtest-corpus-export.ts --rule-id linked_issue_scope_mismatch --output backtest-corpus.json --remote; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::notice::Corpus export from D1 failed (missing or under-scoped CLOUDFLARE_API_TOKEN — it needs D1 read/write on the loopover database). Logic backtest skipped; advisory only, never fails the PR." + fi + + - name: Run logic backtest + id: backtest + if: ${{ steps.corpus.outputs.available == 'true' }} + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + if npx tsx scripts/backtest-logic-check.ts \ + --rule-id linked_issue_scope_mismatch \ + --corpus backtest-corpus.json \ + --head-root . \ + --base-root .advisory-base \ + --output backtest-comment.md \ + --head-sha "$HEAD_SHA" \ + --base-sha "$BASE_SHA" \ + --persist --remote --db loopover \ + --repo "$GITHUB_REPOSITORY" \ + --pr "$PR_NUMBER"; then + echo "ready=true" >> "$GITHUB_OUTPUT" + else + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "::notice::Logic backtest run failed — no comparison produced. Advisory only, never fails the PR." + fi + + # Update-in-place keyed on the comment marker so a re-run edits the existing comment instead of + # stacking a new one per push. + - name: Post or update the backtest PR comment + if: ${{ steps.backtest.outputs.ready == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + post_comment() { + marker="" + # --paginate runs the --jq filter once per page, so pin to the first emitted id. + comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1) + if [ -n "$comment_id" ]; then + gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@backtest-comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@backtest-comment.md + fi + } + if ! post_comment; then + echo "::notice::PR comment post failed — backtest result computed and persisted but not posted. Advisory only, never fails the PR." + fi + + # ============================ REPLAY HALF (#8222) ============================ + # SPEND GUARD (#8222's requirement): the replay costs real model inference, so it is gated on the + # deployment explicitly configuring a budget — the COUNTERFACTUAL_REPLAY_BUDGET repo variable + # (neuron units, see COUNTERFACTUAL_DEFAULT_NEURON_BUDGET) plus the provider endpoint/model. + # #8279: the default provider is Workers AI (the CLOUDFLARE_* secrets above double as its + # credentials — the token needs Workers AI run perms); an explicit COUNTERFACTUAL_OLLAMA_URL + # secret switches to a reachable ollama instead. + - name: Check replay budget configuration + id: budget + env: + REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} + REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} + REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} + run: | + if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ]; then + echo "configured=true" >> "$GITHUB_OUTPUT" + if [ -n "$REPLAY_OLLAMA_URL" ]; then echo "provider=ollama" >> "$GITHUB_OUTPUT"; else echo "provider=workers-ai" >> "$GITHUB_OUTPUT"; fi + else + echo "configured=false" >> "$GITHUB_OUTPUT" + echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables; Workers AI runs on the existing CLOUDFLARE_* secrets, or set COUNTERFACTUAL_OLLAMA_URL to use a reachable ollama). Advisory only, never fails the PR." + fi + + # Cheap self-gate replacing the old workflow's ai-review.ts-only path filter: extract the canonical + # prompt from both checkouts and replay only when the text actually differs — a PR that matched this + # workflow's paths without touching the judge prompt costs two script invocations and stops here. + - name: Extract base/head canonical prompts + if: ${{ steps.budget.outputs.configured == 'true' }} + id: prompts + run: | + if npx tsx scripts/print-review-prompt.ts --root . > head-prompt.txt \ + && npx tsx scripts/print-review-prompt.ts --root .advisory-base > base-prompt.txt \ + && npx tsx scripts/print-review-prompt.ts --root . --version-only > head-version.txt \ + && npx tsx scripts/print-review-prompt.ts --root .advisory-base --version-only > base-version.txt; then + if cmp -s head-prompt.txt base-prompt.txt; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "::notice::Canonical judge prompt text is byte-identical across base/head — nothing to replay." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "::notice::Prompt extraction failed — replay skipped. Advisory only, never fails the PR." + fi + + - name: Export replay corpus from D1 + if: ${{ steps.budget.outputs.configured == 'true' && steps.prompts.outputs.changed == 'true' }} + id: replaycorpus + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + if npx tsx scripts/backtest-corpus-export.ts --rule-id ai_consensus_defect --output replay-corpus.json --remote; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::notice::Corpus export from D1 failed — replay skipped. Advisory only, never fails the PR." + fi + + # Two runs over the IDENTICAL deterministic sample (seed = head SHA, per the design contract's + # per-PR determinism requirement): base prompt first (the baseline artifact), then head with + # --baseline (comparison + comment + persisted run event). + - name: Replay base and head prompts + if: ${{ steps.replaycorpus.outputs.available == 'true' }} + id: replay + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} + REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} + REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} + REPLAY_PROVIDER: ${{ steps.budget.outputs.provider }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + BASE_VERSION=$(cat base-version.txt) + HEAD_VERSION=$(cat head-version.txt) + if npx tsx scripts/counterfactual-replay.ts \ + --fixtures replay-corpus.json \ + --variant "${BASE_VERSION}@${REPLAY_MODEL}" \ + --prompt-file base-prompt.txt \ + --budget "$REPLAY_BUDGET" \ + --seed-suffix "$HEAD_SHA" \ + --provider "$REPLAY_PROVIDER" \ + --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ + --out base-scores.json \ + && npx tsx scripts/counterfactual-replay.ts \ + --fixtures replay-corpus.json \ + --variant "${HEAD_VERSION}@${REPLAY_MODEL}" \ + --prompt-file head-prompt.txt \ + --budget "$REPLAY_BUDGET" \ + --seed-suffix "$HEAD_SHA" \ + --provider "$REPLAY_PROVIDER" \ + --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ + --baseline base-scores.json \ + --base-variant-label "$BASE_VERSION" \ + --comment-out replay-comment.md \ + --head-sha "$HEAD_SHA" --base-sha "$BASE_SHA" \ + --persist --remote --db loopover \ + --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER"; then + echo "ready=true" >> "$GITHUB_OUTPUT" + else + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "::notice::Counterfactual replay failed — no comparison produced. Advisory only, never fails the PR." + fi + + - name: Post or update the replay PR comment + if: ${{ steps.replay.outputs.ready == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + post_comment() { + marker="" + comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1) + if [ -n "$comment_id" ]; then + gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@replay-comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@replay-comment.md + fi + } + if ! post_comment; then + echo "::notice::PR comment post failed — replay computed and persisted but not posted. Advisory only, never fails the PR." + fi + + # The fork half of ci.yml's paired fork==true/!=true convention: fork-originated pull_request runs get + # no repo secrets, so the D1-backed halves cannot run — say so visibly instead of failing or going + # silent. Advisory only either way; a skipped run never blocks anything (#8105). + fork-notice: + name: calibration advisory (skipped for fork PRs) + if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork == true }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Explain the skip + run: echo "::notice::Fork PR — repo secrets are withheld, so the D1-backed calibration advisory checks (logic backtest + counterfactual replay) are skipped. Advisory only; nothing blocks." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b20f9a191..b3dd654b10 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,11 +26,15 @@ jobs: # On push to main everything runs regardless (keeps the coverage baseline solid). changes: name: changes - # Just actions/checkout + git diff --check + dorny/paths-filter -- no build/test work, and it runs FIRST - # to decide whether validate-code needs to run at all, so gating it behind the same contended self-hosted - # queue it's meant to protect was circular (#2507). + # Just actions/checkout + git diff --check + dorny/paths-filter + the diff-scoped dependency review -- + # no build/test work, and it runs FIRST to decide whether validate-code needs to run at all, so gating + # it behind the same contended self-hosted queue it's meant to protect was circular (#2507). runs-on: ubuntu-latest timeout-minutes: 5 + # pull-requests: write is solely for dependency-review's comment-summary-in-pr: on-failure below. + permissions: + contents: read + pull-requests: write outputs: backend: ${{ steps.filter.outputs.backend }} backendConfig: ${{ steps.filter.outputs.backendConfig }} @@ -46,8 +50,8 @@ jobs: controlPlane: ${{ steps.filter.outputs.controlPlane }} observability: ${{ steps.filter.outputs.observability }} steps: - # Debounce rapid re-pushes to an open PR: every job below (validate-code, validate-tests, - # security) has `needs: changes` in its dependency chain, so none of + # Debounce rapid re-pushes to an open PR: every job below (validate-code, validate-tests) + # has `needs: changes` in its dependency chain, so none of # them get queued for a runner until this job finishes -- and this job lives in the SAME PR-ref-scoped # concurrency group as everything else (top of file), so a newer push cancels this job outright while # it's asleep, well before it ever reaches "Filter changed paths". That means a rapid burst of pushes @@ -235,6 +239,19 @@ jobs: - 'test/unit/check-miner-package.test.ts' - 'test/unit/miner-calibration-types.test.ts' + # Diff-scoped security gate, folded in from the old standalone `security` job (2026-07-24, one fewer + # runner slot on every PR at zero added wall-clock -- the API-based lockfile diff takes seconds and + # needs nothing this job hasn't already got). Fails only on vulnerabilities this PR introduces; + # ambient advisories in untouched deps are handled by Renovate + the scheduled audit workflow, so one + # upstream CVE never blocks unrelated PRs. A failure here fails `changes`, which skips the whole + # fan-out and reddens `validate` -- the same end state the standalone job produced, one job earlier. + - name: Dependency review + if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft != true }} + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + fail-on-severity: moderate + comment-summary-in-pr: on-failure + # Path-aware validation. Keep this as one job so a PR uses one dependency # install for the fast checks (drift/typecheck/build/UI toolchain). The # expensive full-suite coverage run is sharded out to validate-tests below @@ -1134,36 +1151,13 @@ jobs: override_pr: ${{ github.event.pull_request.number }} fail_ci_if_error: false - # Diff-scoped security gate: fails only on vulnerabilities this PR introduces. - # Ambient advisories in untouched deps are handled by Renovate + the scheduled - # audit workflow, so one upstream CVE never blocks unrelated PRs. - security: - name: security - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft != true }} - # actions/dependency-review-action needs no self-hosted toolchain/cache (checkout + a lockfile diff), so - # this always runs on ubuntu-latest -- keeping it self-hosted for same-repo PRs only competed for the - # scarce self-hosted pool with validate-code, the one job that actually benefits from it (#2501). - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - pull-requests: write - steps: - - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - - name: Dependency review - uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 - with: - fail-on-severity: moderate - comment-summary-in-pr: on-failure - # Single required status check. Branch protection points at "validate"; this - # aggregates the path-aware jobs and the PR-only dependency review gate so that - # requirement keeps working unchanged. + # aggregates the path-aware jobs (the PR-only dependency review gate lives inside + # `changes` since 2026-07-24) so that requirement keeps working unchanged. # Path-filtered jobs report "skipped", which is treated as success. validate: name: validate - needs: [changes, validate-code, validate-tests, security] + needs: [changes, validate-code, validate-tests] if: ${{ always() }} # Pure result-aggregation (reads needs.*.result, echoes pass/fail) -- no build/test work, so it never # needed the self-hosted pool's cached toolchain (#2507). diff --git a/.github/workflows/counterfactual-replay-check.yml b/.github/workflows/counterfactual-replay-check.yml deleted file mode 100644 index 026b3fa28a..0000000000 --- a/.github/workflows/counterfactual-replay-check.yml +++ /dev/null @@ -1,195 +0,0 @@ -# Counterfactual prompt replay (#8222, sub-epic #8218, epic #8211 track C). When a PR touches the reviewer -# judge-prompt surface, this job replays the CANONICAL judge prompt from BOTH the PR's head and base -# checkouts (scripts/print-review-prompt.ts — the #8139 dual-checkout mechanism) against the recorded -# raw-context fixture corpus, and posts the Pareto-floored comparison as its own advisory comment. Mirrors -# backtest-logic-check.yml's posture end-to-end: separate workflow (untouched PRs pay nothing), advisory -# only (#8105 — never a required check, never blocks merge), every step fails OPEN (notice + green; the -# review engine auto-closes contributor PRs on ANY red check, so this job's own plumbing must never redden). -# -# SPEND GUARD (#8222's requirement): the replay costs real model inference, so the whole run is gated on -# the deployment explicitly configuring a budget — the COUNTERFACTUAL_REPLAY_BUDGET repo variable (neuron -# units, see COUNTERFACTUAL_DEFAULT_NEURON_BUDGET) plus the provider endpoint/model below. Unset (the -# default), the job posts a visible "skipped: no replay budget configured" notice and does nothing else. -name: counterfactual-replay - -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - # Exactly the judge-prompt surface: REVIEW_PROMPT_VERSION + buildSystemPrompt/parseModelReview live - # here. Keep in sync with #8222's spec. - paths: - - "src/services/ai-review.ts" - -permissions: - contents: read - pull-requests: write - -concurrency: - group: counterfactual-replay-${{ github.ref }} - cancel-in-progress: true - -jobs: - replay: - name: counterfactual replay (advisory) - if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork != true }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - # The budget gate comes FIRST so an unconfigured deployment pays one echo, not an npm ci. - # #8279: the default provider is Workers AI (the CLOUDFLARE_* secrets this workflow already holds for - # the corpus export double as its credentials — the token needs Workers AI run perms); an explicit - # COUNTERFACTUAL_OLLAMA_URL secret switches to a reachable ollama instead. - - name: Check replay budget configuration - id: budget - env: - REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} - REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} - REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} - run: | - if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ]; then - echo "configured=true" >> "$GITHUB_OUTPUT" - if [ -n "$REPLAY_OLLAMA_URL" ]; then echo "provider=ollama" >> "$GITHUB_OUTPUT"; else echo "provider=workers-ai" >> "$GITHUB_OUTPUT"; fi - else - echo "configured=false" >> "$GITHUB_OUTPUT" - echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables; Workers AI runs on the existing CLOUDFLARE_* secrets, or set COUNTERFACTUAL_OLLAMA_URL to use a reachable ollama). Advisory only, never fails the PR." - fi - - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - if: ${{ steps.budget.outputs.configured == 'true' }} - with: - persist-credentials: false - - # The PR's base commit inside the head workspace — base-side dynamic imports resolve bare specifiers - # by walking up into the head checkout's node_modules, so one npm ci serves both sides (#8139). - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - if: ${{ steps.budget.outputs.configured == 'true' }} - with: - ref: ${{ github.event.pull_request.base.sha }} - path: .replay-base - persist-credentials: false - - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - if: ${{ steps.budget.outputs.configured == 'true' }} - with: - node-version-file: .nvmrc - cache: "npm" - - - name: Install deps - if: ${{ steps.budget.outputs.configured == 'true' }} - run: npm ci --ignore-scripts - - - name: Build engine package - if: ${{ steps.budget.outputs.configured == 'true' }} - run: npx turbo run build --filter=@loopover/engine - - # Everything below fails OPEN — notice + green, never a red check (#8105 held against our own infra). - - name: Extract base/head canonical prompts - if: ${{ steps.budget.outputs.configured == 'true' }} - id: prompts - run: | - if npx tsx scripts/print-review-prompt.ts --root . > head-prompt.txt \ - && npx tsx scripts/print-review-prompt.ts --root .replay-base > base-prompt.txt \ - && npx tsx scripts/print-review-prompt.ts --root . --version-only > head-version.txt \ - && npx tsx scripts/print-review-prompt.ts --root .replay-base --version-only > base-version.txt; then - if cmp -s head-prompt.txt base-prompt.txt; then - echo "changed=false" >> "$GITHUB_OUTPUT" - echo "::notice::Judge prompt surface touched but the canonical prompt text is byte-identical across base/head — nothing to replay." - else - echo "changed=true" >> "$GITHUB_OUTPUT" - fi - else - echo "changed=false" >> "$GITHUB_OUTPUT" - echo "::notice::Prompt extraction failed — replay skipped. Advisory only, never fails the PR." - fi - - - name: Export corpus from D1 - if: ${{ steps.budget.outputs.configured == 'true' && steps.prompts.outputs.changed == 'true' }} - id: corpus - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - run: | - if npx tsx scripts/backtest-corpus-export.ts --rule-id ai_consensus_defect --output replay-corpus.json --remote; then - echo "available=true" >> "$GITHUB_OUTPUT" - else - echo "available=false" >> "$GITHUB_OUTPUT" - echo "::notice::Corpus export from D1 failed — replay skipped. Advisory only, never fails the PR." - fi - - # Two runs over the IDENTICAL deterministic sample (seed = head SHA, per the design contract's - # per-PR determinism requirement): base prompt first (the baseline artifact), then head with - # --baseline (comparison + comment + persisted run event). - - name: Replay base and head prompts - if: ${{ steps.corpus.outputs.available == 'true' }} - id: replay - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} - REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} - REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} - REPLAY_PROVIDER: ${{ steps.budget.outputs.provider }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - BASE_VERSION=$(cat base-version.txt) - HEAD_VERSION=$(cat head-version.txt) - if npx tsx scripts/counterfactual-replay.ts \ - --fixtures replay-corpus.json \ - --variant "${BASE_VERSION}@${REPLAY_MODEL}" \ - --prompt-file base-prompt.txt \ - --budget "$REPLAY_BUDGET" \ - --seed-suffix "$HEAD_SHA" \ - --provider "$REPLAY_PROVIDER" \ - --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ - --out base-scores.json \ - && npx tsx scripts/counterfactual-replay.ts \ - --fixtures replay-corpus.json \ - --variant "${HEAD_VERSION}@${REPLAY_MODEL}" \ - --prompt-file head-prompt.txt \ - --budget "$REPLAY_BUDGET" \ - --seed-suffix "$HEAD_SHA" \ - --provider "$REPLAY_PROVIDER" \ - --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \ - --baseline base-scores.json \ - --base-variant-label "$BASE_VERSION" \ - --comment-out replay-comment.md \ - --head-sha "$HEAD_SHA" --base-sha "$BASE_SHA" \ - --persist --remote --db loopover \ - --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER"; then - echo "ready=true" >> "$GITHUB_OUTPUT" - else - echo "ready=false" >> "$GITHUB_OUTPUT" - echo "::notice::Counterfactual replay failed — no comparison produced. Advisory only, never fails the PR." - fi - - - name: Post or update the PR comment - if: ${{ steps.replay.outputs.ready == 'true' }} - env: - GH_TOKEN: ${{ github.token }} - PR_NUMBER: ${{ github.event.pull_request.number }} - run: | - post_comment() { - marker="" - comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ - --jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1) - if [ -n "$comment_id" ]; then - gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@replay-comment.md - else - gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@replay-comment.md - fi - } - if ! post_comment; then - echo "::notice::PR comment post failed — replay computed and persisted but not posted. Advisory only, never fails the PR." - fi - - # Fork half of the paired convention: fork runs get no secrets, so the replay cannot run — say so. - fork-notice: - name: counterfactual replay (skipped for fork PRs) - if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork == true }} - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - name: Explain the skip - run: echo "::notice::Fork PR — repo secrets are withheld, so the counterfactual replay is skipped. Advisory only; nothing blocks." diff --git a/.github/workflows/selfhost.yml b/.github/workflows/selfhost.yml index b8e11d5908..774485acd8 100644 --- a/.github/workflows/selfhost.yml +++ b/.github/workflows/selfhost.yml @@ -3,6 +3,14 @@ # 2. Self-host bundle build validation (build-selfhost.ts) # 3. Docker image build + container smoke test (/health, /ready, /metrics) # Unit tests and typecheck are NOT duplicated here — the main CI validate job covers them. +# +# Push-to-main only since 2026-07-24 (was also per-PR): at this repo's PR volume the ~5-minute +# build-boot fired ~100x/week (its path list includes migrations/**, which nearly every backend PR +# touches) and was a top runner-queue consumer. Pre-merge, ci.yml already validates everything except +# the docker boot itself (db:migrations:check, schema drift, the selfhost pg integration suites); a +# boot-breaking merge is caught by this run minutes after landing, and self-host users only ever +# consume tagged releases (release-selfhost.yml), never main -- so the pre-merge boot smoke bought +# almost nothing at real cost. Revisit if a hosted deploy ever tracks main directly. name: self-host on: @@ -19,46 +27,19 @@ on: - "test/unit/selfhost-*" - "test/integration/selfhost-pg*" - ".github/workflows/selfhost.yml" - pull_request: - # Explicit list because the default (opened/synchronize/reopened) omits ready_for_review -- once - # build-boot below skips draft PRs, marking a PR ready must itself trigger a real run (#6670), not - # wait for the next push. Mirrors ci.yml's pull_request.types comment/list exactly. - types: [opened, synchronize, reopened, ready_for_review] - paths: - - "src/selfhost/**" - - "src/server.ts" - - "scripts/build-selfhost.ts" - - "scripts/validate-selfhost-sourcemap.ts" - - "Dockerfile" - - "docker-compose.yml" - - "migrations/**" - - "test/unit/selfhost-*" - - "test/integration/selfhost-pg*" - - ".github/workflows/selfhost.yml" - # Least privilege — the smoke test only reads the repo; no writes, no packages. permissions: contents: read concurrency: - # Same push/pr split as ci.yml's group, and for the same reason: pull requests stay in one ref-scoped - # group so newer commits cancel superseded PR runs (this job boots a Postgres service container, does a - # full npm ci, builds the self-host bundle, and runs two docker buildx builds plus a boot smoke test -- - # letting a superseded push run to completion wastes real minutes). github.sha for push runs keeps - # distinct main commits from cancelling each other's validation. - group: selfhost-${{ github.ref }}-${{ github.event_name == 'push' && github.sha || 'pr' }} + # Push-only workflow: sha-scoped so distinct main commits never cancel each other's validation. + group: selfhost-${{ github.sha }} # NB: keep this a literal boolean, not an expression — see ci.yml for why (startup_failure). cancel-in-progress: true jobs: build-boot: name: build + boot smoke test - # Skip draft PRs (#6670, anti-abuse): this job boots a real Postgres service, does a full npm ci, and - # runs two docker buildx builds plus a container smoke test -- real minutes on every push, including a - # force-push, to a PR nobody has marked ready for review yet. Mirrors ci.yml's validate-code guard; - # draft != true also gates push runs correctly (github.event.pull_request is unset there, so the - # property access evaluates to null, and null != true is true). - if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true }} runs-on: ubuntu-latest timeout-minutes: 20 env: diff --git a/.github/workflows/ui-preview-deploy.yml b/.github/workflows/ui-preview-deploy.yml index 3c020a034e..6b4c3f00fe 100644 --- a/.github/workflows/ui-preview-deploy.yml +++ b/.github/workflows/ui-preview-deploy.yml @@ -1,11 +1,13 @@ name: UI Preview Deploy -# Deploy half of the fork-safe per-PR preview pipeline. Triggered when "UI Preview Build" completes. -# Because it runs on `workflow_run`, GitHub always executes the workflow definition from the DEFAULT -# BRANCH (never the fork's), so it is trusted and may use secrets. It downloads the built `dist` -# artifact (it never checks out or runs PR/fork source — it only uploads the already-built bundle), -# deploys a transient preview version, and records the GitHub Deployment + status that Reviewbot reads -# to render the "after" screenshot. +# FORK-ONLY deploy half of the per-PR preview pipeline (since 2026-07-24 — same-repo PRs deploy +# inline in ui-preview.yml's build-deploy job, and this job skips runner-free for their builds). +# Triggered when "UI Preview Build" completes. Because it runs on `workflow_run`, GitHub always +# executes the workflow definition from the DEFAULT BRANCH (never the fork's), so it is trusted and +# may use secrets. It downloads the built `dist` artifact (the only checkout below is of the default +# branch, solely to invoke the shared local composite action — it never checks out or runs PR/fork +# source), deploys a transient preview version via .github/actions/deploy-ui-preview, and records the +# GitHub Deployment + status that Reviewbot reads to render the "after" screenshot. # # Security boundary: the BUILD ran fork code with NO secrets; this DEPLOY has secrets but runs NO fork # code (wrangler only uploads the bundle — fork code executes solely inside the isolated workers.dev @@ -24,7 +26,7 @@ permissions: contents: read actions: read # download the build artifact from the triggering run deployments: write # record the preview Deployment Reviewbot reads - pull-requests: read # resolve the (often fork) PR for this build by matching head SHA + pull-requests: read # resolve the fork PR for this build by matching head SHA concurrency: group: ui-preview-deploy-${{ github.event.workflow_run.head_sha }} @@ -33,16 +35,18 @@ concurrency: jobs: deploy: name: Deploy UI preview version - # Bind the ONE job that holds Cloudflare credentials to a GitHub deployment environment, so the org - # can attach approval gating and/or environment-scoped secrets to it. Unprotected by default (so - # previews stay automatic); add required reviewers in Settings → Environments → preview to require a - # manual approval before any (fork) preview deploys. + # Bind the ONE fork-facing job that holds Cloudflare credentials to a GitHub deployment environment, + # so the org can attach approval gating and/or environment-scoped secrets to it. Unprotected by + # default (so previews stay automatic); add required reviewers in Settings → Environments → preview + # to require a manual approval before any fork preview deploys. environment: name: preview - url: ${{ steps.upload.outputs.preview_url }} - # Only successful build runs that originated from a pull_request (incl forks). A path-skipped or - # failed build still fires workflow_run with a non-success conclusion — ignore those. - if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' }} + url: ${{ steps.deploy.outputs.preview_url }} + # Only successful build runs that originated from a FORK pull_request: same-repo PRs deploy inline + # in ui-preview.yml (their build workflow still completes and fires this trigger — the repo + # comparison below is what lets those runs skip without consuming a runner). A path-skipped or + # failed build still fires workflow_run with a non-success conclusion — ignore those too. + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name != github.event.workflow_run.repository.full_name }} runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -59,145 +63,37 @@ jobs: echo "::notice::CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID not set — skipping preview deploy (Reviewbot shows before-only)." fi - - name: Setup Node + # TRUSTED checkout of the default branch (github.sha for a workflow_run event is the default + # branch head) — required only so the local `uses: ./.github/actions/deploy-ui-preview` reference + # below can find its action.yml. Never the fork's code. + - name: Checkout default branch if: steps.cfg.outputs.ready == 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: - # Hardcoded, NOT node-version-file: .nvmrc -- this job deliberately has no checkout step (see - # the file header), so there is no .nvmrc on the runner to read. Keep this in sync with .nvmrc's - # pinned version by hand; same exception already established for gittensor-impact.yml and - # visual-capture-fallback.yml. - node-version: "22.23.1" + persist-credentials: false - - name: Install trusted Wrangler - if: steps.cfg.outputs.ready == 'true' - run: npm install --global wrangler@4.95.0 - - # Cross-run download: the artifact lives on the triggering build run, not this one. - - name: Download built UI artifact + - name: Setup Node if: steps.cfg.outputs.ready == 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: - name: ui-preview-dist - path: preview-dist - github-token: ${{ secrets.GITHUB_TOKEN }} - run-id: ${{ github.event.workflow_run.id }} - - # The artifact was produced by the UNTRUSTED build (fork code). Validate it before handing it to - # wrangler: reject symlinks (a path-traversal / exfil vector when the bundle is processed), require - # the expected SSR build structure, and allowlist file extensions so a malicious build can't smuggle - # scripts/binaries/unexpected paths into the deploy. - - name: Validate downloaded artifact - if: steps.cfg.outputs.ready == 'true' - run: | - set -euo pipefail - cd preview-dist - # 1) No symlinks anywhere in the bundle. - symlinks="$(find . -type l)" - if [ -n "$symlinks" ]; then - echo "::error::Artifact contains symlinks — refusing to deploy:" - printf '%s\n' "$symlinks" - exit 1 - fi - # 2) Required SSR build structure (server worker entry + client assets dir). - test -f server/index.mjs || { echo "::error::artifact missing server/index.mjs"; exit 1; } - test -d client || { echo "::error::artifact missing client/ assets dir"; exit 1; } - # 3) Allowlist file extensions — fail on anything that isn't a normal web/build output (blocks - # smuggled scripts/binaries). A few extensionless CF asset files are explicitly permitted. - # `zip` covers the served downloads (e.g. /downloads/loopover-extension.zip) — a passive - # static asset wrangler only uploads (never executes), so allowing it doesn't run fork code. - unexpected="$(find . -regextype posix-extended -type f \ - -not -iregex '.*\.(mjs|js|cjs|map|json|css|html?|txt|svg|png|jpe?g|gif|webp|avif|ico|bmp|woff2?|ttf|otf|eot|wasm|xml|webmanifest|md|csv|zip|wgsl|glb|gltf)$' \ - -not -name '_headers' -not -name '_redirects' -not -name '_routes.json' -not -name '.assetsignore')" - if [ -n "$unexpected" ]; then - echo "::error::Artifact contains unexpected file types — refusing to deploy:" - printf '%s\n' "$unexpected" - exit 1 - fi - echo "Artifact validated: no symlinks, expected SSR structure, allowlisted file types only." - - # The Wrangler config is written HERE (trusted) — never taken from the PR — so a fork cannot - # control bindings, routes, or vars. It points at the downloaded built bundle. It deliberately - # OMITS the production custom-domain route: `wrangler versions upload` creates a 0%-traffic - # preview version (a workers.dev URL) and applies no routes, so the route is unused here — and - # omitting it guarantees that even a future switch to `wrangler deploy` could never point - # fork-built code at the production domain. Do not add a `routes` block to this preview config. - - name: Write trusted preview Wrangler config - if: steps.cfg.outputs.ready == 'true' - run: | - cat > preview-dist/server/wrangler.preview.json <<'JSON' - { - "compatibility_date": "2026-05-28", - "name": "loopover-ui", - "workers_dev": true, - "preview_urls": true, - "compatibility_flags": ["nodejs_compat"], - "placement": { - "mode": "smart" - }, - "observability": { - "enabled": true, - "logs": { - "enabled": true, - "head_sampling_rate": 1 - }, - "traces": { - "enabled": true, - "head_sampling_rate": 1 - } - }, - "vars": { - "VITE_LOOPOVER_API_ORIGIN": "https://api.loopover.ai" - }, - "main": "index.mjs", - "assets": { - "binding": "ASSETS", - "directory": "../client" - }, - "no_bundle": true, - "rules": [ - { - "type": "ESModule", - "globs": ["**/*.mjs", "**/*.js"] - } - ] - } - JSON + # The trusted checkout above restored .nvmrc, so the old hand-synced hardcoded version + # (the no-checkout exception this job used to carry) is no longer needed. + node-version-file: .nvmrc - - name: Upload preview version - id: upload + # Resolve the fork PR this build belongs to BEFORE deploying, so the composite can record the + # deployment against it. Both easy signals are EMPTY for fork PRs (the common case here): + # workflow_run.pull_requests is empty for cross-repo runs, and the commit→PR association API + # doesn't index a fork-head commit from the base repo. So we ALSO match the open PR whose head + # points at this exact build SHA — head.sha is GitHub-set (the base repo tracks the fork head as + # refs/pull/N/head), so it's safe to trust even for forks. + - name: Resolve PR for this build + id: pr if: steps.cfg.outputs.ready == 'true' - env: - CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} - CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - run: | - set -o pipefail - out=$(wrangler versions upload --config preview-dist/server/wrangler.preview.json 2>&1 | tee /dev/stderr) - # Take a workers.dev URL that belongs to the loopover-ui worker, so any other URL in the logs - # (or a changed output format) can't be recorded as the preview by mistake. Tolerant of the - # version-alias prefix (`-loopover-ui..workers.dev`). - url=$(printf '%s\n' "$out" | grep -oiE 'https://[a-z0-9.-]+\.workers\.dev' | grep -i 'loopover-ui' | head -n1) - if [ -z "$url" ]; then - echo "::error::Could not parse an expected loopover-ui preview URL from wrangler output" - exit 1 - fi - echo "preview_url=$url" >> "$GITHUB_OUTPUT" - echo "Preview: $url" - - - name: Record deployment for Reviewbot - if: steps.cfg.outputs.ready == 'true' && steps.upload.outputs.preview_url != '' uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | - const url = ${{ toJSON(steps.upload.outputs.preview_url) }}; - // Trust only the GitHub-set head_sha — never fork-supplied data. - const sha = context.payload.workflow_run.head_sha; + const sha = context.payload.workflow_run.head_sha; // GitHub-set; never fork-supplied const slug = `${context.repo.owner}/${context.repo.repo}`; - // Resolve the PR for this build. Both signals below are EMPTY for fork PRs (the common case - // here): workflow_run.pull_requests is empty for cross-repo runs, and the commit→PR - // association API doesn't index a fork-head commit from the base repo. So we ALSO match the - // open PR whose head points at this exact build SHA — head.sha is GitHub-set (the base repo - // tracks the fork head as refs/pull/N/head), so it's safe to trust even for forks. let prNumber = context.payload.workflow_run.pull_requests?.[0]?.number; if (!prNumber) { const assoc = await github.rest.repos.listPullRequestsAssociatedWithCommit({ @@ -218,67 +114,49 @@ jobs: prNumber = openPrs.find((p) => p.head.sha === sha)?.number; } if (!prNumber) { - core.setFailed(`Could not resolve an open PR for ${sha} — skipping deployment record.`); + core.setFailed(`Could not resolve an open PR for ${sha} — skipping deploy.`); return; } - const deployment = await github.rest.repos.createDeployment({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: sha, - environment: `preview/pr-${prNumber}`, - auto_merge: false, - required_contexts: [], - transient_environment: true, - description: "LoopOver UI preview", - // Reviewbot reads `pr` here to re-review this exact PR once the preview is live. - payload: JSON.stringify({ pr: prNumber, head_sha: sha }), - }); - await github.rest.repos.createDeploymentStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - deployment_id: deployment.data.id, - state: "success", - environment: `preview/pr-${prNumber}`, - environment_url: url, - description: "Preview ready", - }); - core.notice(`Preview deployment recorded for PR #${prNumber}: ${url}`); + core.setOutput("number", String(prNumber)); + + # Cross-run download: the artifact lives on the triggering build run, not this one. + - name: Download built UI artifact + if: steps.cfg.outputs.ready == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ui-preview-dist + path: preview-dist + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ github.event.workflow_run.id }} + + # Validation, trusted config, wrangler upload, and the success Deployment record all live in the + # shared composite (kept byte-identical with ui-preview.yml's same-repo call site by construction). + - name: Deploy preview + id: deploy + if: steps.cfg.outputs.ready == 'true' + uses: ./.github/actions/deploy-ui-preview + with: + pr-number: ${{ steps.pr.outputs.number }} + head-sha: ${{ github.event.workflow_run.head_sha }} + dist-dir: preview-dist + cloudflare-api-token: ${{ secrets.CLOUDFLARE_API_TOKEN }} + cloudflare-account-id: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Record FAILED deployment for Reviewbot # Runs when an earlier step in this job failed (artifact validation rejected the bundle, the # wrangler upload errored, etc.) — i.e. a deploy was attempted but never produced a preview. # Record a `failure` deployment_status so Reviewbot flips the "after" cell from an eternal # spinner to a terminal "preview deploy failed" card, instead of waiting forever for a success - # event that will never come. Gated on CF creds so a credential-less skip records nothing. - if: failure() && steps.cfg.outputs.ready == 'true' + # event that will never come. Gated on CF creds so a credential-less skip records nothing; + # reuses the resolution step's PR (if THAT failed, there is nothing to record against — the + # resolution step already setFailed with the reason). + if: ${{ failure() && steps.cfg.outputs.ready == 'true' && steps.pr.outputs.number != '' }} uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const sha = context.payload.workflow_run.head_sha; // GitHub-set; never fork-supplied - const slug = `${context.repo.owner}/${context.repo.repo}`; - // Same fork-safe PR resolution as the success step above. - let prNumber = context.payload.workflow_run.pull_requests?.[0]?.number; - if (!prNumber) { - const assoc = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: sha, - }); - prNumber = assoc.data.find((p) => p.state === "open" && p.base.repo.full_name === slug)?.number; - } - if (!prNumber) { - const openPrs = await github.paginate(github.rest.pulls.list, { - owner: context.repo.owner, - repo: context.repo.repo, - state: "open", - per_page: 100, - }); - prNumber = openPrs.find((p) => p.head.sha === sha)?.number; - } - if (!prNumber) { - core.warning(`Preview deploy failed but could not resolve a PR for ${sha} — no failure status recorded.`); - return; - } + const prNumber = Number(${{ toJSON(steps.pr.outputs.number) }}); const deployment = await github.rest.repos.createDeployment({ owner: context.repo.owner, repo: context.repo.repo, diff --git a/.github/workflows/ui-preview.yml b/.github/workflows/ui-preview.yml index 8013e16db0..d93814e4f2 100644 --- a/.github/workflows/ui-preview.yml +++ b/.github/workflows/ui-preview.yml @@ -1,19 +1,26 @@ name: UI Preview Build -# Build half of the fork-safe per-PR preview pipeline. Runs for EVERY PR — including forks — and -# deliberately WITHOUT secrets (fork PRs get a read-only token): it only produces the built `dist` -# artifact. The trusted `ui-preview-deploy.yml` (triggered on workflow_run) then deploys that artifact -# WITH secrets and records the GitHub Deployment that Reviewbot reads for the "after" screenshot. +# Per-PR preview pipeline, split by trust (reshaped 2026-07-24 — one fewer runner slot per same-repo +# UI push, which is nearly every UI push at this repo's volume): # -# Why split: a preview requires building the PR's UI, and building runs the PR's (possibly fork-authored) -# code. Doing that here with no secret access — and deploying the resulting bundle in a separate trusted -# step that never executes fork code — is the standard way to give fork PRs previews without exposing -# Cloudflare credentials to untrusted code. +# SAME-REPO PRs (`build-deploy`): a branch in this repo implies a collaborator, and same-repo +# pull_request runs get secrets — so ONE job builds and deploys inline via the shared +# .github/actions/deploy-ui-preview composite. No artifact hop, no second runner, and the +# workflow_run deploy job below skips runner-free. +# +# FORK PRs (`build`): unchanged two-stage boundary. The build runs fork code with NO secrets (fork +# PRs get a read-only token) and only produces the built `dist` artifact; the trusted +# `ui-preview-deploy.yml` (workflow_run) then deploys that artifact WITH secrets and records the +# GitHub Deployment that Reviewbot reads for the "after" screenshot. Building runs the PR's +# (possibly fork-authored) code — doing that with no secret access, and deploying the resulting +# bundle in a separate trusted step that never executes fork code, is the standard way to give fork +# PRs previews without exposing Cloudflare credentials to untrusted code. Do NOT collapse the fork +# path into build-deploy: that boundary is the entire security model. on: pull_request: # Explicit list because the default (opened/synchronize/reopened) omits ready_for_review -- once the - # build job below skips draft PRs, marking a PR ready must itself trigger a real preview build (#6670), + # build jobs below skip draft PRs, marking a PR ready must itself trigger a real preview build (#6670), # not wait for the next push. Mirrors ci.yml's pull_request.types comment/list exactly. types: [opened, synchronize, reopened, ready_for_review] # Scoped to exactly what apps/loopover-ui actually depends on (packages/loopover-ui-kit only) -- @@ -34,16 +41,22 @@ concurrency: cancel-in-progress: true jobs: - build: + # ------------------------- SAME-REPO: build + deploy in one runner ------------------------- + build-deploy: name: Build UI preview artifact - # Skip draft PRs (#6670, anti-abuse): a full npm ci + UI build on every push, including a force-push, to - # a PR nobody has marked ready for review yet. Mirrors ci.yml's validate-code guard; draft != true also - # gates push runs correctly (github.event.pull_request is unset there, so the property access evaluates - # to null, and null != true is true) -- kept even though this workflow has no push trigger today, so the - # condition stays correct if one is ever added. - if: ${{ github.event_name == 'push' || github.event.pull_request.draft != true }} + # Same-repo only (fork PRs take the `build` job below). Skip draft PRs (#6670, anti-abuse): a full + # npm ci + UI build on every push, including a force-push, to a PR nobody has marked ready yet. + if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork != true }} runs-on: ubuntu-latest timeout-minutes: 20 + permissions: + contents: read + deployments: write # record the preview Deployment Reviewbot reads + # Same environment binding as ui-preview-deploy.yml's job: the org can attach approval gating + # and/or environment-scoped secrets to every path that holds Cloudflare credentials. + environment: + name: preview + url: ${{ steps.deploy.outputs.preview_url }} steps: - name: Checkout PR head uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -62,10 +75,10 @@ jobs: # Same steps as the root `ui:build` script, except the last one: that script also builds # @loopover/ui-miner (`npm --workspace @loopover/ui-miner run build`), a full separate Vite app - # this workflow never uploads or deploys (only apps/loopover-ui/dist below) -- every UI PR was - # paying for that build and throwing away the result. `npx turbo run build --filter=@loopover/ui` - # replaces the last two `ui:build` steps (`extension:build && miner-extension:build && ui build` - # already covered by @loopover/ui#build's own turbo.json dependsOn) and skips ui-miner entirely. + # this workflow never uploads or deploys -- every UI PR was paying for that build and throwing + # away the result. `npx turbo run build --filter=@loopover/ui` replaces the last two `ui:build` + # steps (`extension:build && miner-extension:build && ui build` already covered by + # @loopover/ui#build's own turbo.json dependsOn) and skips ui-miner entirely. - name: Build UI env: VITE_LOOPOVER_API_ORIGIN: https://api.loopover.ai @@ -76,6 +89,96 @@ jobs: VITE_PREVIEW: "1" run: npm run ui:kit:build && npx turbo run build --filter=@loopover/engine && npm run ui:openapi && npx turbo run build --filter=@loopover/ui + # Deploys stay best-effort when the Cloudflare secrets aren't configured (Reviewbot shows + # before-only) — mirrors ui-preview-deploy.yml's guard. + - name: Check Cloudflare secrets + id: cfg + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + if [ -n "$CLOUDFLARE_API_TOKEN" ] && [ -n "$CLOUDFLARE_ACCOUNT_ID" ]; then + echo "ready=true" >> "$GITHUB_OUTPUT" + else + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "::notice::CLOUDFLARE_API_TOKEN / CLOUDFLARE_ACCOUNT_ID not set — skipping preview deploy (Reviewbot shows before-only)." + fi + + - name: Deploy preview + id: deploy + if: ${{ steps.cfg.outputs.ready == 'true' }} + uses: ./.github/actions/deploy-ui-preview + with: + pr-number: ${{ github.event.pull_request.number }} + # GitHub-set; the same value the checkout above pinned. + head-sha: ${{ github.event.pull_request.head.sha }} + dist-dir: apps/loopover-ui/dist + cloudflare-api-token: ${{ secrets.CLOUDFLARE_API_TOKEN }} + cloudflare-account-id: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Record FAILED deployment for Reviewbot + # A deploy was attempted but never produced a preview (build-side failures skip this via the + # cfg gate ordering: cfg only runs after a successful build). Record a `failure` + # deployment_status so Reviewbot flips the "after" cell from an eternal spinner to a terminal + # "preview deploy failed" card. Gated on CF creds so a credential-less skip records nothing. + if: ${{ failure() && steps.cfg.outputs.ready == 'true' }} + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const sha = context.payload.pull_request.head.sha; // GitHub-set; never fork-supplied + const prNumber = context.payload.pull_request.number; + const deployment = await github.rest.repos.createDeployment({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: sha, + environment: `preview/pr-${prNumber}`, + auto_merge: false, + required_contexts: [], + transient_environment: true, + description: "LoopOver UI preview (failed)", + payload: JSON.stringify({ pr: prNumber, head_sha: sha }), + }); + await github.rest.repos.createDeploymentStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + deployment_id: deployment.data.id, + state: "failure", + environment: `preview/pr-${prNumber}`, + description: "Preview deploy failed", + }); + core.notice(`Recorded FAILED preview deployment for PR #${prNumber}.`); + + # ------------------------- FORK: unprivileged build half only ------------------------- + build: + name: Build UI preview artifact (fork) + # Fork PRs only — no secrets here, ever (see the file header). Draft guard as above. + if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork == true }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout PR head + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + run: npm ci + + # Same build as build-deploy above — see its comment for the turbo-filter rationale. + - name: Build UI + env: + VITE_LOOPOVER_API_ORIGIN: https://api.loopover.ai + VITE_PREVIEW: "1" + run: npm run ui:kit:build && npx turbo run build --filter=@loopover/engine && npm run ui:openapi && npx turbo run build --filter=@loopover/ui + # The trusted deploy workflow downloads this by name + run-id. It contains only the built bundle # (server/ + client/) — no secrets, no source needed downstream. - name: Upload built UI artifact diff --git a/scripts/backtest-logic-check.ts b/scripts/backtest-logic-check.ts index cf76d79261..ace8fd155f 100644 --- a/scripts/backtest-logic-check.ts +++ b/scripts/backtest-logic-check.ts @@ -1,6 +1,6 @@ #!/usr/bin/env node // Logic/regex-change backtest CLI (#8139, epic #8082) — the CI-side runner .github/workflows/ -// backtest-logic-check.yml invokes. Loads a corpus manifest (produced by backtest-corpus-export.ts), +// calibration-advisory.yml invokes. Loads a corpus manifest (produced by backtest-corpus-export.ts), // dynamically imports the registered detection function from TWO checkout roots (the PR's head and its base // — the dual checkout is what makes an honest before/after comparison possible), runs the pure core's // scoring/comparison, writes the PR-comment Markdown, and optionally persists the run to D1 via diff --git a/test/unit/ci-skip-draft-prs.test.ts b/test/unit/ci-skip-draft-prs.test.ts index fbecec7dea..7635ae29a2 100644 --- a/test/unit/ci-skip-draft-prs.test.ts +++ b/test/unit/ci-skip-draft-prs.test.ts @@ -14,7 +14,7 @@ function record(value: unknown, label: string): Record { } // Regression guard: draft PRs were consuming the exact same heavy GitHub-hosted-runner fan-out as ready -// PRs (validate-code, the full-suite validate-tests coverage run, security), which both +// PRs (validate-code, the full-suite validate-tests coverage run), which both // (a) let contributors farm bot labels/AI review/screenshots for free while sitting in draft, and (b) // materially worsened the account's shared-runner queue contention for every other PR in flight. The heavy // jobs now require the PR not be a draft; the cheap `changes` and `validate` (result-aggregation) jobs still @@ -40,14 +40,19 @@ describe("ci.yml skips the heavy jobs for draft pull requests", () => { }, ); - it("security's if-condition requires github.event.pull_request.draft != true", () => { - const job = record(jobs.security, "jobs.security"); - expect(String(job.if)).toBe("${{ github.event_name == 'pull_request' && github.event.pull_request.draft != true }}"); + it("the dependency-review step (folded into changes, 2026-07-24) skips draft PRs and push runs", () => { + const job = record(jobs.changes, "jobs.changes"); + const steps = (job.steps as Array>).filter((s) => s.name === "Dependency review"); + expect(steps).toHaveLength(1); + expect(String(steps[0]?.if)).toBe("${{ github.event_name == 'pull_request' && github.event.pull_request.draft != true }}"); + expect(String((steps[0]?.uses as string) ?? "")).toContain("actions/dependency-review-action@"); + // The standalone job must stay gone -- a resurrected `security` job would silently double the check. + expect(jobs.security).toBeUndefined(); }); - it("validate still aggregates all three gated jobs and treats a skipped dependency as success", () => { + it("validate still aggregates the gated jobs and treats a skipped dependency as success", () => { const job = record(jobs.validate, "jobs.validate"); - expect(job.needs).toEqual(["changes", "validate-code", "validate-tests", "security"]); + expect(job.needs).toEqual(["changes", "validate-code", "validate-tests"]); expect(String(job.if)).toBe("${{ always() }}"); }); }); diff --git a/test/unit/workflow-runner-labels.test.ts b/test/unit/workflow-runner-labels.test.ts index 268c846259..6ca8905074 100644 --- a/test/unit/workflow-runner-labels.test.ts +++ b/test/unit/workflow-runner-labels.test.ts @@ -17,7 +17,7 @@ describe("workflow runner labels", () => { expect(workflow).not.toContain("|| 'self-hosted'"); expect(workflow).not.toContain('"fork-ci"'); expect(workflow).toContain("validate-code:"); - expect(workflow).toContain("needs: [changes, validate-code, validate-tests, security]"); + expect(workflow).toContain("needs: [changes, validate-code, validate-tests]"); expect(workflow).not.toContain("\n lint:\n"); expect(workflow).not.toContain("\n test:\n"); expect(workflow).not.toContain("\n workers:\n"); @@ -32,10 +32,8 @@ describe("workflow runner labels", () => { // validate-tests (#ci-shard-coverage) is the unsharded full-suite coverage run, split out of // validate-code so the long suite never serializes with the much-faster checks (unsharded again // 2026-07 -- see the job's header comment in ci.yml; the old merge job is gone with the shards). - const validateTestsJob = workflow.slice(workflow.indexOf("\n validate-tests:\n"), workflow.indexOf("\n security:\n")); + const validateTestsJob = workflow.slice(workflow.indexOf("\n validate-tests:\n"), workflow.indexOf("\n validate:\n")); expect(validateTestsJob).toContain("runs-on: ubuntu-latest"); - const securityJob = workflow.slice(workflow.indexOf("\n security:\n"), workflow.indexOf("\n validate:\n")); - expect(securityJob).toContain("runs-on: ubuntu-latest"); const validateJob = workflow.slice(workflow.indexOf("\n validate:\n")); expect(validateJob).toContain("runs-on: ubuntu-latest"); }); @@ -50,11 +48,9 @@ describe("workflow runner labels", () => { it("cancels a superseded selfhost.yml run instead of letting it run to completion (#2496)", () => { const workflow = read(".github/workflows/selfhost.yml"); - // Same push/pr split as ci.yml's own group, for the same reason: distinct main-branch pushes must not - // cancel each other's validation, only a superseded run on the SAME ref/PR should be cancelled. - expect(workflow).toContain( - "group: selfhost-${{ github.ref }}-${{ github.event_name == 'push' && github.sha || 'pr' }}", - ); + // Push-only workflow since 2026-07-24: sha-scoped so distinct main-branch pushes never cancel each + // other's validation (the old push/pr split went with the pull_request trigger). + expect(workflow).toContain("group: selfhost-${{ github.sha }}"); expect(workflow).toContain("cancel-in-progress: true"); // Must be a literal boolean, not an expression -- ci.yml's own comment documents that an expression here // causes GitHub to fail the workflow at startup (startup_failure).