From 85a34ff04154786c7080f2cc6a510d78c17830e6 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sat, 24 Oct 2026 16:09:04 -0400 Subject: [PATCH 1/2] fix(ci): log pr-feedback-dispatch's rerun-ci-status and post-verify-notes API readings Both jobs spent github-token calls without posting an api-budget notice, so GitHub Support's 18:57-19:57Z window counted their runs (87 of the workflow's 91 executing runs) among the unattributed spend. - rerun-ci-status: the sparse checkout adds the log-api-budget library; read_api keeps each response's reading (refusals included) through api_budget_read, and the job ends with the log-api-budget step. A failed checkout leaves no library and the job runs as before. - post-verify-notes: the artifact listing's gh api call runs through api_budget_wrap_gh, and the job ends with the log-api-budget step. Call counts per path are unchanged; scripts/ci-rerun-ci-status.test.sh now pins one reading per call on every counted path. Co-Authored-By: Claude Opus 5.5 --- .../log-api-budget/log-api-budget.test.mjs | 52 ++++++++++++++++++- .github/workflows/pr-feedback-dispatch.yml | 42 +++++++++++++-- scripts/ci-rerun-ci-status.test.sh | 41 +++++++++++---- 3 files changed, 119 insertions(+), 16 deletions(-) diff --git a/.github/actions/log-api-budget/log-api-budget.test.mjs b/.github/actions/log-api-budget/log-api-budget.test.mjs index 2a650dfffa4..10ab1f33970 100644 --- a/.github/actions/log-api-budget/log-api-budget.test.mjs +++ b/.github/actions/log-api-budget/log-api-budget.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { execFileSync, spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; @@ -184,3 +184,53 @@ test("ci-status carries api-budget.bash verbatim in its API_BUDGET_SH env", () = const carried = block[1].replace(/^ {8}/gm, "").replace(/\n+$/, "\n"); assert.equal(carried, readFileSync(LIB, "utf8")); }); + +// pr-feedback-dispatch.yml's post-verify-notes and rerun-ci-status jobs post +// no other notice, so without these lines their calls go unattributed. +const FEEDBACK = new URL("../../workflows/pr-feedback-dispatch.yml", import.meta.url); + +// The job's steps, from ` :` to the next job. +function jobText(job) { + const lines = readFileSync(FEEDBACK, "utf8").split("\n"); + const start = lines.indexOf(` ${job}:`); + assert.notEqual(start, -1, job); + const end = lines.findIndex((line, i) => i > start && /^ {2}[a-z][\w-]*:$/.test(line)); + return lines.slice(start + 1, end === -1 ? undefined : end).join("\n"); +} + +test("pr-feedback-dispatch.yml: each job that calls the API ends by logging its github-token readings", () => { + for (const job of ["post-verify-notes", "dispatch", "rerun-ci-status"]) { + const steps = jobText(job).split("\n - name: "); + assert.equal( + steps.at(-1).split("\n").filter((line, i) => i === 0 || line.startsWith(" ")).join("\n"), + "Log the API budget at the end\n if: always()\n continue-on-error: true\n timeout-minutes: 1\n" + + " uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository]\n with:\n tokens: github-token", + job, + ); + } +}); + +test("pr-feedback-dispatch.yml: post-verify-notes keeps the reading of its artifact listing and decides as before", () => + withDir((dir) => { + const step = jobText("post-verify-notes").split("\n - name: ").find((text) => text.startsWith("Download the fail verdict")); + const body = /\n {8}run: \|\n((?: {10}[^\n]*\n|\n)+)/.exec(`${step}\n`)[1].replace(/^ {10}/gm, ""); + for (const sub of [".base/.github/actions/log-api-budget", "bin", "rt"]) { + mkdirSync(join(dir, sub), { recursive: true }); + } + copyFileSync(LIB, join(dir, ".base/.github/actions/log-api-budget/api-budget.bash")); + // A gh that answers `api -i --jq ` with a header block and no FAIL name. + writeFileSync( + join(dir, "bin/gh"), + `#!/usr/bin/env bash\n[[ $1 == api && $2 == -i && $3 == repos/o/r/actions/runs/77/artifacts* && $4 == --jq ]] || { echo "args: $*" >&2; exit 9; }\n` + + `printf '${head({ used: 41 }).replace(/\r/g, "\\r").replace(/\n/g, "\\n")}'\n`, + ); + chmodSync(join(dir, "bin/gh"), 0o755); + const result = spawnSync("bash", ["-e", "-c", body], { + cwd: dir, + encoding: "utf8", + env: { PATH: `${join(dir, "bin")}:${process.env.PATH}`, RUNNER_TEMP: join(dir, "rt"), RUN_ID: "77", GITHUB_REPOSITORY: "o/r", GITHUB_OUTPUT: join(dir, "out") }, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, "::notice::run 77 uploaded no fail verdict; nothing to post\n"); + assert.equal(state(join(dir, "rt"), "github-token"), "200 5000 4959 41 1792748042 core\n"); + })); diff --git a/.github/workflows/pr-feedback-dispatch.yml b/.github/workflows/pr-feedback-dispatch.yml index f3eadb6c644..0c3f44d671e 100644 --- a/.github/workflows/pr-feedback-dispatch.yml +++ b/.github/workflows/pr-feedback-dispatch.yml @@ -120,6 +120,10 @@ jobs: echo "::error::the upstream run id is not a run id" exit 1 fi + # Each `gh api` call keeps its rate-limit reading for the job-end + # api-budget notice (.github/actions/log-api-budget). + # shellcheck source=/dev/null + { . .base/.github/actions/log-api-budget/api-budget.bash && api_budget_wrap_gh github-token; } 2>/dev/null || true # Every FAIL-shaped name, then the ones the readers take (the # grammar babysit_verify_evidence.NAME_RE holds, at most 194 # characters, issues ascending and once each). A FAIL-shaped name @@ -148,6 +152,14 @@ jobs: VERDICT_FILE: ${{ runner.temp }}/verdict/verdict.json run: node .base/.github/scripts/pr-unblock/verify-notes.mjs + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + dispatch: # A review run GitHub skipped (a draft PR) has reviewed nothing, and a # cancelled one (most often superseded by a newer push) finished no review. @@ -306,8 +318,10 @@ jobs: # too; one that # started earlier and is still in progress waits for its run, polled every # 60 s for at most 10 minutes, then is re-run. No model, and no checkout - # but the trusted-actors list: the token can only read contents, runs and - # the PR and re-run a job. + # but the trusted-actors list and the api-budget library: the token can + # only read contents, runs and the PR and re-run a job. Each read's + # rate-limit reading is logged at the end of the job + # (.github/actions/log-api-budget). # # The re-run's attempt is started by github-actions[bot], which pr-verify's # trigger gate does not list, so just before the re-run the job uploads @@ -335,8 +349,9 @@ jobs: pull-requests: read # the PR's state and live head steps: # The list the record check reads, from the default branch, never the - # PR head. A failed checkout leaves no list: ci-status is still re-run, - # unrecorded unless the review run is a first attempt. + # PR head, and the api-budget library. A failed checkout leaves no list: + # ci-status is still re-run, unrecorded unless the review run is a + # first attempt, and no reading is logged. - name: Check out the trusted-actors list continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -346,6 +361,7 @@ jobs: persist-credentials: false sparse-checkout: | /.github/standards/trusted-actors/ + /.github/actions/log-api-budget/ sparse-checkout-cone-mode: false - name: Find the ci-status to re-run @@ -362,7 +378,15 @@ jobs: REVIEW_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} REVIEW_RERUNNER_ID: ${{ github.event.workflow_run.triggering_actor.id }} TRUSTED_ACTORS: .base/.github/standards/trusted-actors/trusted-actors.json + API_BUDGET_SH: .base/.github/actions/log-api-budget/api-budget.bash run: | + # shellcheck source=/dev/null + . "$API_BUDGET_SH" 2>/dev/null || true + # budget : keeps its rate-limit reading for the + # job-end api-budget notice; nothing without the library. + budget() { + ! declare -F api_budget_read >/dev/null || api_budget_read github-token /dev/stdin <<<"$1" >/dev/null + } if [[ ! $HEAD_SHA =~ ^[0-9a-f]{40}$ ]]; then echo "::error::the review run's head '$HEAD_SHA' is not a commit SHA." exit 1 @@ -399,9 +423,11 @@ jobs: local attempt out hdr status left reset after wait for attempt in 1 2 3; do if out=$(gh api -i "$3"); then + budget "$out" printf -v "$1" '%s' "$(sed '1,/^\r\?$/d' <<<"$out")" return 0 fi + budget "$out" hdr=$(sed '/^\r\?$/q' <<<"$out") status=$(awk 'NR == 1 { print $2 }' <<<"$hdr") read -r left reset after < <(tr -d '\r' <<<"$hdr" | awk -F': *' '{ h[tolower($1)] = $2 } @@ -549,3 +575,11 @@ jobs: fi gh api -X POST "repos/$GITHUB_REPOSITORY/actions/jobs/$JOB_ID/rerun" >/dev/null echo "Re-ran ci-status (job $JOB_ID)." + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token diff --git a/scripts/ci-rerun-ci-status.test.sh b/scripts/ci-rerun-ci-status.test.sh index 6038339190f..c1aebce6508 100755 --- a/scripts/ci-rerun-ci-status.test.sh +++ b/scripts/ci-rerun-ci-status.test.sh @@ -14,7 +14,8 @@ # closed PR, a moved head, no run yet, a ci-status not started or one started # after both lanes finished is left alone. A rate limit waits for its reset # within the cap, or fails past it. Unreadable reads fail. Each -# path's API call count is pinned. +# path's API call count is pinned, and each response's rate-limit reading is +# kept for the job-end api-budget notice. # test-scope: .github/workflows/pr-feedback-dispatch.yml set -uo pipefail @@ -57,7 +58,8 @@ fi for env in 'HEAD_SHA: ${{ github.event.workflow_run.head_sha }}' 'PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }}' \ 'LANE_PATH: ${{ github.event.workflow_run.path }}' 'LANE_DONE_AT: ${{ github.event.workflow_run.updated_at }}' "POLL_LIMIT_SECONDS: '600'" \ 'REVIEW_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}' 'REVIEW_RERUNNER_ID: ${{ github.event.workflow_run.triggering_actor.id }}' \ - 'TRUSTED_ACTORS: .base/.github/standards/trusted-actors/trusted-actors.json'; do + 'TRUSTED_ACTORS: .base/.github/standards/trusted-actors/trusted-actors.json' \ + 'API_BUDGET_SH: .base/.github/actions/log-api-budget/api-budget.bash'; do if [[ "$step" == *"$env"* ]]; then ok "step env carries $env"; else fail "step env lacks $env"; fi done job="$(awk '/^ rerun-ci-status:$/ { on = 1; next } on && /^ [a-z]/ { exit } on { print }' "$WORKFLOW")" @@ -71,14 +73,14 @@ fi list_step="$(step_of 'Check out the trusted-actors list')" # shellcheck disable=SC2016 # the ${{ }} text is workflow source, not shell for want in 'continue-on-error: true' 'uses: actions/checkout@' 'ref: refs/heads/main' 'path: .base' 'persist-credentials: false' \ - '/.github/standards/trusted-actors/' 'sparse-checkout-cone-mode: false'; do + '/.github/standards/trusted-actors/' '/.github/actions/log-api-budget/' 'sparse-checkout-cone-mode: false'; do if [[ "$list_step" == *"$want"* ]]; then ok "the list checkout carries '$want'"; else fail "the list checkout lacks '$want'"; fi done # The record goes up between the find and the re-run, and each later step -# runs only on the find step's outputs. -order="$(grep -nE '^ - name: (Find the ci-status to re-run|Record the re-run|Re-run ci-status)$' <<<"$job" | cut -d: -f3- | paste -sd'|')" -if [[ "$order" == ' Find the ci-status to re-run| Record the re-run| Re-run ci-status' ]]; then - ok "the job finds, records, then re-runs" +# runs only on the find step's outputs. The job ends by logging its readings. +order="$(grep -nE '^ - name: ' <<<"$job" | cut -d: -f3- | paste -sd'|')" +if [[ "$order" == ' Check out the trusted-actors list| Find the ci-status to re-run| Record the re-run| Re-run ci-status| Log the API budget at the end' ]]; then + ok "the job finds, records, re-runs, then logs the API budget" else fail "step order: '$order'" fi @@ -95,6 +97,10 @@ done for want in "if: steps.find.outputs.job-id != ''" 'JOB_ID: ${{ steps.find.outputs.job-id }}'; do if [[ "$rerun_step" == *"$want"* ]]; then ok "the re-run step carries '$want'"; else fail "the re-run step lacks '$want'"; fi done +budget_step="$(step_of 'Log the API budget at the end')" +for want in 'if: always()' 'continue-on-error: true' 'uses: ./.base/.github/actions/log-api-budget' 'tokens: github-token'; do + if [[ "$budget_step" == *"$want"* ]]; then ok "the budget step carries '$want'"; else fail "the budget step lacks '$want'"; fi +done # The stub logs its arguments. A pulls read prints STUB_PR. A review lane's # runs read prints STUB_LANE_RUNS and a pr-require-checks runs read STUB_RUNS, @@ -183,11 +189,14 @@ expect() { : >"$TMP_ROOT/log" : >"$TMP_ROOT/out" rm -f "$TMP_ROOT/record.json" + rm -rf "$TMP_ROOT/rt" + mkdir -p "$TMP_ROOT/rt" out="$(PATH="$TMP_ROOT/bin:$PATH" STUB_LOG="$TMP_ROOT/log" GITHUB_REPOSITORY=o/r HEAD_SHA="${HEAD_SHA-$HEAD}" \ PR_NUMBER="${PR_NUMBER-42}" LANE_PATH="${LANE_PATH-$CODE}" LANE_DONE_AT="${LANE_DONE_AT-$DONE}" \ GITHUB_OUTPUT="$TMP_ROOT/out" RECORD_FILE="$TMP_ROOT/record.json" GITHUB_RUN_ID=555 \ REVIEW_ATTEMPT="${REVIEW_ATTEMPT-1}" REVIEW_RERUNNER_ID="${REVIEW_RERUNNER_ID-$KYLE}" \ - TRUSTED_ACTORS="${TRUSTED_ACTORS-$TMP_ROOT/trusted-actors.json}" POLL_SECONDS=60 POLL_LIMIT_SECONDS=600 bash -e "$TMP_ROOT/body.sh" 2>&1)" && rc=0 || rc=$? + TRUSTED_ACTORS="${TRUSTED_ACTORS-$TMP_ROOT/trusted-actors.json}" RUNNER_TEMP="$TMP_ROOT/rt" \ + API_BUDGET_SH="${API_BUDGET_SH-$ROOT/.github/actions/log-api-budget/api-budget.bash}" POLL_SECONDS=60 POLL_LIMIT_SECONDS=600 bash -e "$TMP_ROOT/body.sh" 2>&1)" && rc=0 || rc=$? if [[ "$rc" -ne "$want_rc" ]]; then fail "$label: expected rc=$want_rc got rc=$rc out='$out'" elif [[ -n "${3:-}" && "$out" != *"$3"* ]]; then @@ -212,11 +221,13 @@ record=${3:-rerun-ci-status-9-2-$HEAD}" fail "$2: outputs: $(paste -sd'|' "$TMP_ROOT/out"), posts: $(grep -c '^api -X POST' "$TMP_ROOT/log")" fi } -# The API calls the last case made, against the count its path is pinned to. +# The API calls the last case made, against the count its path is pinned to, +# and one rate-limit reading kept for each of them. check_calls() { - local n + local n r n=$(grep -c '^api ' "$TMP_ROOT/log") - if [[ "$n" == "$1" ]]; then ok "$2 ($1 API calls)"; else fail "$2: $n API calls, want $1: $(paste -sd'|' "$TMP_ROOT/log")"; fi + r=$(grep -c . "$TMP_ROOT/rt/api-budget/github-token" 2>/dev/null) + if [[ "$n" == "$1" && "$r" == "$1" ]]; then ok "$2 ($1 API calls, $1 readings)"; else fail "$2: $n API calls and $r readings, want $1: $(paste -sd'|' "$TMP_ROOT/log")"; fi } # Both lanes finished: the head's last lane re-runs ci-status once. @@ -346,6 +357,8 @@ echo 1 >"$STUB_LIMIT_FILE" expect "a rate-limited read waits for the reset, then re-runs" 0 "Rate limited reading the pr-require-checks runs on $HEAD; waiting 120s for the reset." check_reran 1 "the read after the reset goes on to the re-run" if [[ "$(grep -c '^sleep 120$' "$TMP_ROOT/log")" == 1 ]]; then ok "it slept until one second past the reset"; else fail "sleeps: $(grep '^sleep' "$TMP_ROOT/log" | paste -sd'|')"; fi +check_calls 5 "the refused read is counted and its reading kept" +if grep -qx '403 na 0 na 1119 na' "$TMP_ROOT/rt/api-budget/github-token"; then ok "the refusal's reading names its status and reset"; else fail "readings: $(paste -sd'|' "$TMP_ROOT/rt/api-budget/github-token")"; fi echo 1 >"$STUB_LIMIT_FILE" STUB_RETRY_AFTER=30 expect "a retry-after refusal waits that many seconds" 0 "waiting 30s for the reset." check_reran 1 "the read after retry-after goes on to the re-run" @@ -414,6 +427,12 @@ check_unrecorded "a missing review attempt hands on no record" REVIEW_ATTEMPT=1 REVIEW_RERUNNER_ID=$STRANGER TRUSTED_ACTORS="$TMP_ROOT/absent.json" expect "a first review attempt is recorded without the list" 0 "as attempt 2." check_reran 1 "a first attempt hands on the record" +# Without the library (a failed checkout) the job runs as before and keeps no reading. +STUB_PR="$(pr)" STUB_LANE_RUNS="$(runs "$(run 3 completed "$HEAD" "$SEC")")" STUB_RUNS="$(runs "$(run 9)")" STUB_JOBS="$(cijob completed)" +REVIEW_ATTEMPT=1 API_BUDGET_SH="$TMP_ROOT/absent.bash" expect "a missing api-budget library still re-runs" 0 "Re-running ci-status (job 77) of run 9" +check_reran 1 "the re-run is handed on without the library" +if [[ ! -e "$TMP_ROOT/rt/api-budget/github-token" ]]; then ok "no reading is kept without the library"; else fail "readings kept without the library"; fi + # The re-run step re-runs only a job id, once. rerun_expect() { local label="$1" want_rc="$2" out rc From 6f8c834b369ed262242ac7ef868283f3cab2f617 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:07:42 -0400 Subject: [PATCH 2/2] test(ci): say which post-verify-notes call leaves an api-budget reading Only the artifact listing leaves one. gh run download passes through the gh wrapper but is not gh api, and verify-notes.mjs uses its own fetch client, so the job-end notice understates the job's spend. Co-Authored-By: Claude Opus 5.5 --- .github/actions/log-api-budget/log-api-budget.test.mjs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/actions/log-api-budget/log-api-budget.test.mjs b/.github/actions/log-api-budget/log-api-budget.test.mjs index f837a6cb6e3..b95a756e747 100644 --- a/.github/actions/log-api-budget/log-api-budget.test.mjs +++ b/.github/actions/log-api-budget/log-api-budget.test.mjs @@ -373,9 +373,10 @@ test("ci-status carries api-budget.bash verbatim in its API_BUDGET_SH env", () = // pr-feedback-dispatch.yml's post-verify-notes and rerun-ci-status jobs post // no other notice, so without these lines their calls go unattributed. // post-verify-notes' notice covers only its artifact listing, the one call -// through the gh wrapper: `gh run download` is not `gh api`, and -// verify-notes.mjs calls through address-feedback's createGitHubApi, a plain -// fetch that keeps no reading. That notice understates the job's spend. +// that leaves a reading: `gh run download` goes through the gh wrapper but is +// not `gh api`, so it leaves none, and verify-notes.mjs calls through +// address-feedback's createGitHubApi, a plain fetch that keeps no reading. +// That notice understates the job's spend. const FEEDBACK = new URL("../../workflows/pr-feedback-dispatch.yml", import.meta.url); // The job's steps, from ` :` to the next job.