From eb11370293c0bad001655c677d51335ec797dd8d Mon Sep 17 00:00:00 2001 From: prql-bot <107324867+prql-bot@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:03:33 +0000 Subject: [PATCH 1/2] ci: bound tend job runtime with `timeout-minutes` --- .config/tend.yaml | 42 +++++++++++++++++++++++ .github/workflows/tend-ci-fix.yaml | 1 + .github/workflows/tend-mention.yaml | 1 + .github/workflows/tend-nightly.yaml | 1 + .github/workflows/tend-notifications.yaml | 1 + .github/workflows/tend-review-runs.yaml | 1 + .github/workflows/tend-review.yaml | 1 + .github/workflows/tend-triage.yaml | 1 + .github/workflows/tend-weekly.yaml | 1 + 9 files changed, 50 insertions(+) diff --git a/.config/tend.yaml b/.config/tend.yaml index 69751644747d..76bf1658cb0d 100644 --- a/.config/tend.yaml +++ b/.config/tend.yaml @@ -10,7 +10,49 @@ secrets: - DOCKERHUB_USERNAME setup: - uses: ./.github/actions/tend-setup +# Without these, every tend job is bounded only by GitHub's 360-minute default +# job cap. `tend-nightly` run 26438780303 hit exactly that on 2026-05-26 (07:32Z +# → 13:32Z, six hours, no `tend-outage` row filed). A hang in the setup steps is +# worse still: those run before the action is entered, so the harness supervisor +# never sees them and the job cap is the only bound that reaches them. +# +# The caps below are ~2x the slowest run observed over the last 100 completed +# runs of each workflow; see the PR that added them for the table. They sit below +# the supervisor's `timeout_seconds` (350m), so they are the operative session +# budget — deliberate, since under `cancel-in-progress: false` an unbounded +# review holds the PR's review queue behind it. workflows: ci-fix: watched_workflows: - tests + jobs: + fix-ci: + timeout-minutes: 60 + review: + jobs: + review: + timeout-minutes: 120 + mention: + jobs: + handle: + timeout-minutes: 120 + triage: + jobs: + triage: + timeout-minutes: 120 + review-runs: + jobs: + review-runs: + timeout-minutes: 120 + nightly: + jobs: + nightly: + timeout-minutes: 180 + weekly: + jobs: + weekly: + timeout-minutes: 60 + notifications: + jobs: + notifications: + timeout-minutes: 60 diff --git a/.github/workflows/tend-ci-fix.yaml b/.github/workflows/tend-ci-fix.yaml index c80978496c52..13dbd6e2e2b9 100644 --- a/.github/workflows/tend-ci-fix.yaml +++ b/.github/workflows/tend-ci-fix.yaml @@ -46,3 +46,4 @@ jobs: - Run URL: ${{ github.event.workflow_run.html_url }} - Commit: ${{ github.event.workflow_run.head_sha }} - Commit message: ${{ github.event.workflow_run.head_commit.message }} + timeout-minutes: 60 diff --git a/.github/workflows/tend-mention.yaml b/.github/workflows/tend-mention.yaml index f38c9eb073be..9a4d73b57b1f 100644 --- a/.github/workflows/tend-mention.yaml +++ b/.github/workflows/tend-mention.yaml @@ -477,3 +477,4 @@ jobs: || format('pulls/comments/{0}', github.event.comment.id || github.event.client_payload.id) }} BOT_NAME: prql-bot GITHUB_TOKEN: ${{ secrets.TEND_BOT_TOKEN }} + timeout-minutes: 120 diff --git a/.github/workflows/tend-nightly.yaml b/.github/workflows/tend-nightly.yaml index 7c97bd10d986..803be0a2dd85 100644 --- a/.github/workflows/tend-nightly.yaml +++ b/.github/workflows/tend-nightly.yaml @@ -43,3 +43,4 @@ jobs: model: opus prompt: | /tend-ci-runner:nightly + timeout-minutes: 180 diff --git a/.github/workflows/tend-notifications.yaml b/.github/workflows/tend-notifications.yaml index 4451a265ab0c..5406852bbfca 100644 --- a/.github/workflows/tend-notifications.yaml +++ b/.github/workflows/tend-notifications.yaml @@ -158,3 +158,4 @@ jobs: model: opus prompt: | /tend-ci-runner:notifications + timeout-minutes: 60 diff --git a/.github/workflows/tend-review-runs.yaml b/.github/workflows/tend-review-runs.yaml index 2a6ef405e5d6..efd2fa84c486 100644 --- a/.github/workflows/tend-review-runs.yaml +++ b/.github/workflows/tend-review-runs.yaml @@ -43,3 +43,4 @@ jobs: model: opus prompt: | /tend-ci-runner:review-runs + timeout-minutes: 120 diff --git a/.github/workflows/tend-review.yaml b/.github/workflows/tend-review.yaml index 49918c44b318..13ede268881a 100644 --- a/.github/workflows/tend-review.yaml +++ b/.github/workflows/tend-review.yaml @@ -181,3 +181,4 @@ jobs: TARGET: issues/${{ github.event.pull_request.number }} BOT_NAME: prql-bot GITHUB_TOKEN: ${{ secrets.TEND_BOT_TOKEN }} + timeout-minutes: 120 diff --git a/.github/workflows/tend-triage.yaml b/.github/workflows/tend-triage.yaml index 124e3bb4b364..e41992acf96d 100644 --- a/.github/workflows/tend-triage.yaml +++ b/.github/workflows/tend-triage.yaml @@ -71,3 +71,4 @@ jobs: TARGET: issues/${{ github.event.issue.number }} BOT_NAME: prql-bot GITHUB_TOKEN: ${{ secrets.TEND_BOT_TOKEN }} + timeout-minutes: 120 diff --git a/.github/workflows/tend-weekly.yaml b/.github/workflows/tend-weekly.yaml index 4a318276c6b5..239a96cb9d7c 100644 --- a/.github/workflows/tend-weekly.yaml +++ b/.github/workflows/tend-weekly.yaml @@ -43,3 +43,4 @@ jobs: model: opus prompt: | /tend-ci-runner:weekly + timeout-minutes: 60 From 9fc8fd5da3d2f3e907d49dda95b1103179404f82 Mon Sep 17 00:00:00 2001 From: prql-bot <107324867+prql-bot@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:15:03 +0000 Subject: [PATCH 2/2] ci: cap `relay` and `verify` in tend-mention too --- .config/tend.yaml | 8 ++++++++ .github/workflows/tend-mention.yaml | 2 ++ 2 files changed, 10 insertions(+) diff --git a/.config/tend.yaml b/.config/tend.yaml index 76bf1658cb0d..a288a612522a 100644 --- a/.config/tend.yaml +++ b/.config/tend.yaml @@ -34,6 +34,14 @@ workflows: timeout-minutes: 120 mention: jobs: + # `relay` and `verify` are one-step script jobs that have never exceeded + # 15s across the last 40 runs, but `handle` declares `needs: verify`, so a + # wedged `verify` holds a mention unanswered for the full six hours without + # ever reaching the capped job. + relay: + timeout-minutes: 10 + verify: + timeout-minutes: 10 handle: timeout-minutes: 120 triage: diff --git a/.github/workflows/tend-mention.yaml b/.github/workflows/tend-mention.yaml index 9a4d73b57b1f..2b03332fbc3d 100644 --- a/.github/workflows/tend-mention.yaml +++ b/.github/workflows/tend-mention.yaml @@ -88,6 +88,7 @@ jobs: -f "client_payload[kind]=${{ github.event_name }}" \ -f "client_payload[pr]=${{ github.event.pull_request.number }}" \ -f "client_payload[id]=${{ github.event.review.id || github.event.comment.id }}" + timeout-minutes: 10 verify: # Skip comments on the issues tend files about its own health: the action @@ -340,6 +341,7 @@ jobs: PAYLOAD_KIND: ${{ github.event.client_payload.kind }} PAYLOAD_PR: ${{ github.event.client_payload.pr }} PAYLOAD_ID: ${{ github.event.client_payload.id }} + timeout-minutes: 10 handle: needs: verify