diff --git a/.github/actions/log-api-budget/action.yml b/.github/actions/log-api-budget/action.yml index d0863463f1a..14adac8f67c 100644 --- a/.github/actions/log-api-budget/action.yml +++ b/.github/actions/log-api-budget/action.yml @@ -9,8 +9,8 @@ description: > counter every concurrent job shares) is not, and M its 304 answers, which GitHub does not count. Then one line for the job's content-creating writes (count and most in any 60 s). Also writes each line and a `-cost=N` - line to the step summary. A token with no reading gets no line. Never fails - the job. + line to the step summary. A token with no reading gets only its api-calls + line, at 0. Never fails the job. inputs: tokens: @@ -39,14 +39,16 @@ runs: for name in $TOKENS; do [[ $name =~ ^[a-z][a-z0-9-]{0,39}$ ]] || { echo "::warning::api-budget: skipped a token name that is not lower-case kebab case"; continue; } lines=$(api_budget_emit "$name") - [[ -n $lines ]] || continue - printf '%s\n' "$lines" - sed 's/^::notice title=api-budget::/api-budget: /' <<<"$lines" >>"$GITHUB_STEP_SUMMARY" + if [[ -n $lines ]]; then + printf '%s\n' "$lines" + sed 's/^::notice title=api-budget::/api-budget: /' <<<"$lines" >>"$GITHUB_STEP_SUMMARY" + fi # This job's own calls, one reading each: unlike cost, no other job's. - # A notice, as the hub reads annotations, not the log. + # A notice, as the hub reads annotations, not the log; 0 with no reading. calls=$(api_budget_emit_calls "$name") printf '%s\n' "$calls" echo "api-calls ${calls#*title=api-calls::}" >>"$GITHUB_STEP_SUMMARY" + [[ -n $lines ]] || continue cost=${lines##*cost=} costs+=("$name-cost=${cost/na/unknown}") done diff --git a/.github/actions/log-api-budget/api-budget.bash b/.github/actions/log-api-budget/api-budget.bash index ef0d35abef6..81c6ae6ef21 100644 --- a/.github/actions/log-api-budget/api-budget.bash +++ b/.github/actions/log-api-budget/api-budget.bash @@ -71,10 +71,10 @@ # api_budget_emit_content the notice for the job's content-creating # writes: count and most in any 60 s # api_budget_emit_calls ... -# the hub's api-calls notice for each token with -# a reading, `=N not-modified=M`: N is -# the job's own calls, M its 304 answers, which -# GitHub does not count +# the hub's api-calls notice for each token, +# `=N not-modified=M`: N is the job's own +# calls, M its 304 answers, which GitHub does not +# count; 0 for a token with no reading # # pr-require-checks.yml's ci-status job carries this file verbatim in its # API_BUDGET_SH env (it has no checkout of this directory); @@ -377,7 +377,8 @@ api_budget_emit_calls() { local token file for token; do file=${RUNNER_TEMP:-}/api-budget/$token - [[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ && -s $file ]] || continue + [[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ ]] || continue + [[ -s $file ]] || file=/dev/null awk '$1 != 304 { n++ } $1 == 304 { m++ } END { printf "::notice title=api-calls::%s=%d not-modified=%d\n", token, n, m }' token="$token" "$file" done return 0 diff --git a/.github/actions/log-api-budget/log-api-budget.test.mjs b/.github/actions/log-api-budget/log-api-budget.test.mjs index c73303261d3..3f2fdb1539d 100644 --- a/.github/actions/log-api-budget/log-api-budget.test.mjs +++ b/.github/actions/log-api-budget/log-api-budget.test.mjs @@ -156,7 +156,7 @@ test("recordReading appends the same line shape, na for a missing or malformed v assert.equal(existsSync(join(dir, "api-budget", "lanes-app")), false); })); -test("the action emits each named token, the call count as an api-calls notice the hub reads, the summary and the costs output, and reads no /rate_limit", () => +test("the action emits each named token, the call count as an api-calls notice the hub reads (0 with no reading), the summary and the costs output, and reads no /rate_limit", () => withDir((dir) => { // Two calls GitHub counted, then a conditional GET answered 304. readEach(dir, "github-token", head({ used: 13 }), head({ used: 41 }), head({ status: 304, used: 41 })); @@ -177,12 +177,14 @@ test("the action emits each named token, the call count as an api-calls notice t }); assert.equal( out, - `${NOTICE}token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` + + "::notice title=api-calls::lanes-app=0 not-modified=0\n" + + `${NOTICE}token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` + "::notice title=api-calls::github-token=2 not-modified=1\n", ); assert.equal( readFileSync(join(dir, "summary"), "utf8"), - `api-budget: token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` + + "api-calls lanes-app=0 not-modified=0\n" + + `api-budget: token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` + "api-calls github-token=2 not-modified=1\ngithub-token-cost=28\n", ); assert.equal(readFileSync(join(dir, "output"), "utf8"), "costs=github-token-cost=28\n"); @@ -203,9 +205,12 @@ test("ci-status ends with the same api-budget and api-calls notices the action w `${NOTICE}token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` + "::notice title=api-calls::github-token=2 not-modified=1\n", ); - // With no reading, no line, as the action skips a token with none. + // With no reading, no api-budget line and an api-calls count of 0, as the action writes. rmSync(join(dir, "api-budget"), { recursive: true }); - assert.equal(execFileSync("bash", ["-c", block[1]], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir, API_BUDGET_SH: readFileSync(LIB, "utf8") } }), ""); + assert.equal( + execFileSync("bash", ["-c", block[1]], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir, API_BUDGET_SH: readFileSync(LIB, "utf8") } }), + "::notice title=api-calls::github-token=0 not-modified=0\n", + ); })); // The shell port of the rate guard decides every shared vector as @@ -633,8 +638,11 @@ test("the action writes the content-creation notice after the token lines", () = TOKENS: "github-token", }, }); - assert.equal(out, `${NOTICE}content-creation ${WHERE} count=1 max-per-minute=1\n`); - assert.equal(readFileSync(join(dir, "summary"), "utf8"), `api-budget: content-creation ${WHERE} count=1 max-per-minute=1\n`); + assert.equal(out, `::notice title=api-calls::github-token=0 not-modified=0\n${NOTICE}content-creation ${WHERE} count=1 max-per-minute=1\n`); + assert.equal( + readFileSync(join(dir, "summary"), "utf8"), + `api-calls github-token=0 not-modified=0\napi-budget: content-creation ${WHERE} count=1 max-per-minute=1\n`, + ); })); // Every lane job that calls GitHub through the rate guard names its token and @@ -671,6 +679,31 @@ test("each heavy lane job names its API_BUDGET_TOKEN and ends with the log-api-b } }); +// Jobs that hold the job token but call GitHub only through actions the +// counter cannot see still end by logging it, so each run carries its +// api-calls line, 0 when no step of the job's own called GitHub. +test("each job that holds the job token and calls no guarded client ends with the log-api-budget step", () => { + const windows = ["select-tests-windows", ...["rename-1", "rename-2", "secrets", "guards", "hooks", "lib", "worktree"].map((s) => `test-windows-${s}`), "test-macos-planning-shared"]; + const jobs = { + "pr-test-windows.yml": Object.fromEntries(windows.map((job) => [job, "./"])), + "post-merge-detect-silent-revert.yml": { "detect-silent-revert": "./" }, + "release-publish-plugins.yml": { merge: "./", build: "./" }, + }; + for (const [file, byJob] of Object.entries(jobs)) { + const text = readFileSync(new URL(`../../workflows/${file}`, import.meta.url), "utf8"); + for (const [job, root] of Object.entries(byJob)) { + const block = new RegExp(`\\n ${job}:\\n([\\s\\S]*?)(?=\\n [a-z0-9-]+:\\n|$)`).exec(text)?.[1] ?? ""; + const last = block.split(/\n {6}- name: /).at(-1).replace(/(\n *(#[^\n]*)?)+$/, ""); + assert.equal( + last, + "Log the API budget at the end\n if: always()\n continue-on-error: true\n timeout-minutes: 1\n" + + ` uses: ${root}.github/actions/log-api-budget # zizmor: ignore[self-repository]\n with:\n tokens: github-token`, + `${file} ${job}`, + ); + } + } +}); + // 950 used of a header limit of 5000 leaves 50 of a planned 1000, under a // floor of 100; without the cap the same reading is far over its floor. test("gh guard: API_BUDGET_LIMIT_CAP plans the floor to the cap, and only the header's remaining stops", () => diff --git a/.github/scripts/pr-merge/workflow.test.mjs b/.github/scripts/pr-merge/workflow.test.mjs index 0c7a91f4ce1..993a2f49e4a 100644 --- a/.github/scripts/pr-merge/workflow.test.mjs +++ b/.github/scripts/pr-merge/workflow.test.mjs @@ -500,7 +500,7 @@ test("enable-auto-merge logs both tokens' readings at job end, before the token // sees. Every such job carries the log-api-budget step, run whatever failed // or stopped before it, naming github-token: under `always()` alone, so a job // that ends before it mints a lane token still logs the job token's calls. A -// token with no reading gets no line (log-api-budget.test.mjs). +// token with no reading logs 0 (log-api-budget.test.mjs). test("every job that calls GitHub with the job token logs its api-calls", () => { const ids = [...jobs.matchAll(/\n {2}([a-z-]+):\n/g)].map((m) => m[1]); const callers = ids.filter((id) => job(id).includes("${{ github.token }}")); @@ -649,7 +649,7 @@ for (const jobId of ["merge", "enable-auto-merge"]) { env: { ...base, GITHUB_ACTION_PATH: fileURLToPath(new URL("../../actions/log-api-budget/", import.meta.url)), TOKENS: /\n {10}tokens: (.*)\n/.exec(log)[1] }, }); assert.match(out.stdout, /^::notice title=api-calls::github-token=3 not-modified=0$/m); - assert.doesNotMatch(out.stdout, /lanes-app/); + assert.match(out.stdout, /^::notice title=api-calls::lanes-app=0 not-modified=0$/m); } finally { server.close(); rmSync(dir, { recursive: true, force: true }); diff --git a/.github/workflows/post-merge-detect-silent-revert.yml b/.github/workflows/post-merge-detect-silent-revert.yml index f8120c00c36..a3c3accb46e 100644 --- a/.github/workflows/post-merge-detect-silent-revert.yml +++ b/.github/workflows/post-merge-detect-silent-revert.yml @@ -133,7 +133,7 @@ jobs: # exited 0 with the content gone: it answered the other question. Two # re-lands each missed it, and a hand audit found it, not this lane. # - # ORDER IS LOAD-BEARING, and this step is deliberately LAST. + # ORDER IS LOAD-BEARING, and this step is deliberately the LAST check. # # A step with no status function carries an implicit `success() &&`, so a # step that fails SKIPS every later step in the job. Placed before the @@ -162,3 +162,11 @@ jobs: - name: Assert the recorded incidents' content is restored if: success() || failure() run: scripts/check-silent-revert.sh --verify-restoration + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token diff --git a/.github/workflows/pr-require-checks.yml b/.github/workflows/pr-require-checks.yml index 5bc8c179b82..7ddb4ce48c0 100644 --- a/.github/workflows/pr-require-checks.yml +++ b/.github/workflows/pr-require-checks.yml @@ -3366,10 +3366,10 @@ jobs: # api_budget_emit_content the notice for the job's content-creating # writes: count and most in any 60 s # api_budget_emit_calls ... - # the hub's api-calls notice for each token with - # a reading, `=N not-modified=M`: N is - # the job's own calls, M its 304 answers, which - # GitHub does not count + # the hub's api-calls notice for each token, + # `=N not-modified=M`: N is the job's own + # calls, M its 304 answers, which GitHub does not + # count; 0 for a token with no reading # # pr-require-checks.yml's ci-status job carries this file verbatim in its # API_BUDGET_SH env (it has no checkout of this directory); @@ -3672,7 +3672,8 @@ jobs: local token file for token; do file=${RUNNER_TEMP:-}/api-budget/$token - [[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ && -s $file ]] || continue + [[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ ]] || continue + [[ -s $file ]] || file=/dev/null awk '$1 != 304 { n++ } $1 == 304 { m++ } END { printf "::notice title=api-calls::%s=%d not-modified=%d\n", token, n, m }' token="$token" "$file" done return 0 diff --git a/.github/workflows/pr-test-windows.yml b/.github/workflows/pr-test-windows.yml index 50b36160077..87617011c78 100644 --- a/.github/workflows/pr-test-windows.yml +++ b/.github/workflows/pr-test-windows.yml @@ -93,6 +93,14 @@ jobs: [ -n "$base" ] || echo "::notice::A $GITHUB_EVENT_NAME run with no usable base runs every Windows step." scripts/plan-test-lanes.sh ${base:+--base "$base"} >>"$GITHUB_OUTPUT" + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + # Every suite runs on Linux too. A suite is here because a Linux runner cannot # exercise part of what it tests: # @@ -206,6 +214,14 @@ jobs: if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/guardrails/hooks/run-guards.test.sh') run: bash plugins/guardrails/hooks/run-guards.test.sh + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + test-windows-rename-2: needs: [select-tests-windows] if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-rename-2') }} @@ -253,6 +269,14 @@ jobs: shell: pwsh run: pwsh -NoProfile -NonInteractive -File plugins/machine-health/skills/audit/scripts/run-tests.ps1 + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + test-windows-secrets: needs: [select-tests-windows] if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-secrets') }} @@ -290,6 +314,14 @@ jobs: working-directory: plugins/disk-hygiene/skills/clean/scripts run: python -m unittest -v test_hygiene.GuardTests + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + test-windows-guards: needs: [select-tests-windows] if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-guards') }} @@ -311,6 +343,14 @@ jobs: if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/guardrails/hooks/hardcoded-path-check.test.sh') run: bash plugins/guardrails/hooks/hardcoded-path-check.test.sh + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + test-windows-hooks: needs: [select-tests-windows] if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-hooks') }} @@ -373,6 +413,14 @@ jobs: if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/developer-experience/skills/setup/scripts/pointer-line.test.sh') run: bash plugins/developer-experience/skills/setup/scripts/pointer-line.test.sh + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + test-windows-lib: needs: [select-tests-windows] if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-lib') }} @@ -430,6 +478,14 @@ jobs: $config.Output.Verbosity = 'Detailed' Invoke-Pester -Configuration $config + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + test-windows-worktree: needs: [select-tests-windows] if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-worktree') }} @@ -477,6 +533,14 @@ jobs: if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/planning/shared/tests/test_server.py') run: python -m unittest discover -v -s plugins/planning/shared/tests -p 'test_tunnel.py' + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + # The planning shared-session suites on macOS, the third platform # plugins/planning/shared/README.md names. detached.py's POSIX branches, file # modes and the bash-run fake cloudflared run here as on Linux, but on Darwin: @@ -539,3 +603,11 @@ jobs: if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/planning/shared/tests/test_tunnel.py') timeout-minutes: 3 run: python -m unittest discover -v -s plugins/planning/shared/tests -p 'test_tunnel.py' + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token diff --git a/.github/workflows/release-publish-plugins.yml b/.github/workflows/release-publish-plugins.yml index 40af823d2f9..83cb02a3bbe 100644 --- a/.github/workflows/release-publish-plugins.yml +++ b/.github/workflows/release-publish-plugins.yml @@ -139,6 +139,14 @@ jobs: echo "Merged release pull request #$PR_NUMBER at $HEAD_SHA." fi + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + build: needs: [merge] if: ${{ !cancelled() && github.ref == 'refs/heads/main' && ((inputs.action || 'merge') != 'merge' || needs.merge.outputs.rebuild == 'true') }} @@ -324,3 +332,11 @@ jobs: printf '\n%s\n' "$line" >>"$body" gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" fi + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token