Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions .github/actions/log-api-budget/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ description: >
counter every concurrent job shares) is not, and M its 304 answers, which
GitHub does not count. Then one line for the job's content-creating writes
(count and most in any 60 s). Also writes each line and a `<name>-cost=N`
line to the step summary. A token with no reading gets no line. Never fails
the job.
line to the step summary. A token with no reading gets only its api-calls
line, at 0. Never fails the job.

inputs:
tokens:
Expand Down Expand Up @@ -39,14 +39,16 @@ runs:
for name in $TOKENS; do
[[ $name =~ ^[a-z][a-z0-9-]{0,39}$ ]] || { echo "::warning::api-budget: skipped a token name that is not lower-case kebab case"; continue; }
lines=$(api_budget_emit "$name")
[[ -n $lines ]] || continue
printf '%s\n' "$lines"
sed 's/^::notice title=api-budget::/api-budget: /' <<<"$lines" >>"$GITHUB_STEP_SUMMARY"
if [[ -n $lines ]]; then
printf '%s\n' "$lines"
sed 's/^::notice title=api-budget::/api-budget: /' <<<"$lines" >>"$GITHUB_STEP_SUMMARY"
fi
# This job's own calls, one reading each: unlike cost, no other job's.
# A notice, as the hub reads annotations, not the log.
# A notice, as the hub reads annotations, not the log; 0 with no reading.
calls=$(api_budget_emit_calls "$name")
printf '%s\n' "$calls"
echo "api-calls ${calls#*title=api-calls::}" >>"$GITHUB_STEP_SUMMARY"
[[ -n $lines ]] || continue
cost=${lines##*cost=}
costs+=("$name-cost=${cost/na/unknown}")
done
Expand Down
11 changes: 6 additions & 5 deletions .github/actions/log-api-budget/api-budget.bash
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,10 @@
# api_budget_emit_content the notice for the job's content-creating
# writes: count and most in any 60 s
# api_budget_emit_calls <token>...
# the hub's api-calls notice for each token with
# a reading, `<token>=N not-modified=M`: N is
# the job's own calls, M its 304 answers, which
# GitHub does not count
# the hub's api-calls notice for each token,
# `<token>=N not-modified=M`: N is the job's own
# calls, M its 304 answers, which GitHub does not
# count; 0 for a token with no reading
#
# pr-require-checks.yml's ci-status job carries this file verbatim in its
# API_BUDGET_SH env (it has no checkout of this directory);
Expand Down Expand Up @@ -377,7 +377,8 @@ api_budget_emit_calls() {
local token file
for token; do
file=${RUNNER_TEMP:-}/api-budget/$token
[[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ && -s $file ]] || continue
[[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ ]] || continue
[[ -s $file ]] || file=/dev/null
awk '$1 != 304 { n++ } $1 == 304 { m++ } END { printf "::notice title=api-calls::%s=%d not-modified=%d\n", token, n, m }' token="$token" "$file"
done
return 0
Expand Down
47 changes: 40 additions & 7 deletions .github/actions/log-api-budget/log-api-budget.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ test("recordReading appends the same line shape, na for a missing or malformed v
assert.equal(existsSync(join(dir, "api-budget", "lanes-app")), false);
}));

test("the action emits each named token, the call count as an api-calls notice the hub reads, the summary and the costs output, and reads no /rate_limit", () =>
test("the action emits each named token, the call count as an api-calls notice the hub reads (0 with no reading), the summary and the costs output, and reads no /rate_limit", () =>
withDir((dir) => {
// Two calls GitHub counted, then a conditional GET answered 304.
readEach(dir, "github-token", head({ used: 13 }), head({ used: 41 }), head({ status: 304, used: 41 }));
Expand All @@ -177,12 +177,14 @@ test("the action emits each named token, the call count as an api-calls notice t
});
assert.equal(
out,
`${NOTICE}token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` +
"::notice title=api-calls::lanes-app=0 not-modified=0\n" +
`${NOTICE}token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` +
"::notice title=api-calls::github-token=2 not-modified=1\n",
);
assert.equal(
readFileSync(join(dir, "summary"), "utf8"),
`api-budget: token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` +
"api-calls lanes-app=0 not-modified=0\n" +
`api-budget: token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` +
"api-calls github-token=2 not-modified=1\ngithub-token-cost=28\n",
);
assert.equal(readFileSync(join(dir, "output"), "utf8"), "costs=github-token-cost=28\n");
Expand All @@ -203,9 +205,12 @@ test("ci-status ends with the same api-budget and api-calls notices the action w
`${NOTICE}token=github-token ${WHERE} limit=5000 remaining=4959 used=41 reset=1792748042 resource=core cost=28\n` +
"::notice title=api-calls::github-token=2 not-modified=1\n",
);
// With no reading, no line, as the action skips a token with none.
// With no reading, no api-budget line and an api-calls count of 0, as the action writes.
rmSync(join(dir, "api-budget"), { recursive: true });
assert.equal(execFileSync("bash", ["-c", block[1]], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir, API_BUDGET_SH: readFileSync(LIB, "utf8") } }), "");
assert.equal(
execFileSync("bash", ["-c", block[1]], { encoding: "utf8", env: { PATH: process.env.PATH, RUNNER_TEMP: dir, API_BUDGET_SH: readFileSync(LIB, "utf8") } }),
"::notice title=api-calls::github-token=0 not-modified=0\n",
);
}));

// The shell port of the rate guard decides every shared vector as
Expand Down Expand Up @@ -633,8 +638,11 @@ test("the action writes the content-creation notice after the token lines", () =
TOKENS: "github-token",
},
});
assert.equal(out, `${NOTICE}content-creation ${WHERE} count=1 max-per-minute=1\n`);
assert.equal(readFileSync(join(dir, "summary"), "utf8"), `api-budget: content-creation ${WHERE} count=1 max-per-minute=1\n`);
assert.equal(out, `::notice title=api-calls::github-token=0 not-modified=0\n${NOTICE}content-creation ${WHERE} count=1 max-per-minute=1\n`);
assert.equal(
readFileSync(join(dir, "summary"), "utf8"),
`api-calls github-token=0 not-modified=0\napi-budget: content-creation ${WHERE} count=1 max-per-minute=1\n`,
);
}));

// Every lane job that calls GitHub through the rate guard names its token and
Expand Down Expand Up @@ -671,6 +679,31 @@ test("each heavy lane job names its API_BUDGET_TOKEN and ends with the log-api-b
}
});

// Jobs that hold the job token but call GitHub only through actions the
// counter cannot see still end by logging it, so each run carries its
// api-calls line, 0 when no step of the job's own called GitHub.
test("each job that holds the job token and calls no guarded client ends with the log-api-budget step", () => {
const windows = ["select-tests-windows", ...["rename-1", "rename-2", "secrets", "guards", "hooks", "lib", "worktree"].map((s) => `test-windows-${s}`), "test-macos-planning-shared"];
const jobs = {
"pr-test-windows.yml": Object.fromEntries(windows.map((job) => [job, "./"])),
"post-merge-detect-silent-revert.yml": { "detect-silent-revert": "./" },
"release-publish-plugins.yml": { merge: "./", build: "./" },
};
for (const [file, byJob] of Object.entries(jobs)) {
const text = readFileSync(new URL(`../../workflows/${file}`, import.meta.url), "utf8");
for (const [job, root] of Object.entries(byJob)) {
const block = new RegExp(`\\n ${job}:\\n([\\s\\S]*?)(?=\\n [a-z0-9-]+:\\n|$)`).exec(text)?.[1] ?? "";
const last = block.split(/\n {6}- name: /).at(-1).replace(/(\n *(#[^\n]*)?)+$/, "");
assert.equal(
last,
"Log the API budget at the end\n if: always()\n continue-on-error: true\n timeout-minutes: 1\n" +
` uses: ${root}.github/actions/log-api-budget # zizmor: ignore[self-repository]\n with:\n tokens: github-token`,
`${file} ${job}`,
);
}
}
});

// 950 used of a header limit of 5000 leaves 50 of a planned 1000, under a
// floor of 100; without the cap the same reading is far over its floor.
test("gh guard: API_BUDGET_LIMIT_CAP plans the floor to the cap, and only the header's remaining stops", () =>
Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/pr-merge/workflow.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ test("enable-auto-merge logs both tokens' readings at job end, before the token
// sees. Every such job carries the log-api-budget step, run whatever failed
// or stopped before it, naming github-token: under `always()` alone, so a job
// that ends before it mints a lane token still logs the job token's calls. A
// token with no reading gets no line (log-api-budget.test.mjs).
// token with no reading logs 0 (log-api-budget.test.mjs).
test("every job that calls GitHub with the job token logs its api-calls", () => {
const ids = [...jobs.matchAll(/\n {2}([a-z-]+):\n/g)].map((m) => m[1]);
const callers = ids.filter((id) => job(id).includes("${{ github.token }}"));
Expand Down Expand Up @@ -649,7 +649,7 @@ for (const jobId of ["merge", "enable-auto-merge"]) {
env: { ...base, GITHUB_ACTION_PATH: fileURLToPath(new URL("../../actions/log-api-budget/", import.meta.url)), TOKENS: /\n {10}tokens: (.*)\n/.exec(log)[1] },
});
assert.match(out.stdout, /^::notice title=api-calls::github-token=3 not-modified=0$/m);
assert.doesNotMatch(out.stdout, /lanes-app/);
assert.match(out.stdout, /^::notice title=api-calls::lanes-app=0 not-modified=0$/m);
} finally {
server.close();
rmSync(dir, { recursive: true, force: true });
Expand Down
10 changes: 9 additions & 1 deletion .github/workflows/post-merge-detect-silent-revert.yml
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ jobs:
# exited 0 with the content gone: it answered the other question. Two
# re-lands each missed it, and a hand audit found it, not this lane.
#
# ORDER IS LOAD-BEARING, and this step is deliberately LAST.
# ORDER IS LOAD-BEARING, and this step is deliberately the LAST check.
#
# A step with no status function carries an implicit `success() &&`, so a
# step that fails SKIPS every later step in the job. Placed before the
Expand Down Expand Up @@ -162,3 +162,11 @@ jobs:
- name: Assert the recorded incidents' content is restored
if: success() || failure()
run: scripts/check-silent-revert.sh --verify-restoration

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
11 changes: 6 additions & 5 deletions .github/workflows/pr-require-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3366,10 +3366,10 @@ jobs:
# api_budget_emit_content the notice for the job's content-creating
# writes: count and most in any 60 s
# api_budget_emit_calls <token>...
# the hub's api-calls notice for each token with
# a reading, `<token>=N not-modified=M`: N is
# the job's own calls, M its 304 answers, which
# GitHub does not count
# the hub's api-calls notice for each token,
# `<token>=N not-modified=M`: N is the job's own
# calls, M its 304 answers, which GitHub does not
# count; 0 for a token with no reading
#
# pr-require-checks.yml's ci-status job carries this file verbatim in its
# API_BUDGET_SH env (it has no checkout of this directory);
Expand Down Expand Up @@ -3672,7 +3672,8 @@ jobs:
local token file
for token; do
file=${RUNNER_TEMP:-}/api-budget/$token
[[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ && -s $file ]] || continue
[[ -n ${RUNNER_TEMP:-} && $token =~ ^[a-z][a-z0-9-]{0,39}$ ]] || continue
[[ -s $file ]] || file=/dev/null
awk '$1 != 304 { n++ } $1 == 304 { m++ } END { printf "::notice title=api-calls::%s=%d not-modified=%d\n", token, n, m }' token="$token" "$file"
done
return 0
Expand Down
72 changes: 72 additions & 0 deletions .github/workflows/pr-test-windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,14 @@ jobs:
[ -n "$base" ] || echo "::notice::A $GITHUB_EVENT_NAME run with no usable base runs every Windows step."
scripts/plan-test-lanes.sh ${base:+--base "$base"} >>"$GITHUB_OUTPUT"

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

# Every suite runs on Linux too. A suite is here because a Linux runner cannot
# exercise part of what it tests:
#
Expand Down Expand Up @@ -206,6 +214,14 @@ jobs:
if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/guardrails/hooks/run-guards.test.sh')
run: bash plugins/guardrails/hooks/run-guards.test.sh

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

test-windows-rename-2:
needs: [select-tests-windows]
if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-rename-2') }}
Expand Down Expand Up @@ -253,6 +269,14 @@ jobs:
shell: pwsh
run: pwsh -NoProfile -NonInteractive -File plugins/machine-health/skills/audit/scripts/run-tests.ps1

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

test-windows-secrets:
needs: [select-tests-windows]
if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-secrets') }}
Expand Down Expand Up @@ -290,6 +314,14 @@ jobs:
working-directory: plugins/disk-hygiene/skills/clean/scripts
run: python -m unittest -v test_hygiene.GuardTests

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

test-windows-guards:
needs: [select-tests-windows]
if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-guards') }}
Expand All @@ -311,6 +343,14 @@ jobs:
if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/guardrails/hooks/hardcoded-path-check.test.sh')
run: bash plugins/guardrails/hooks/hardcoded-path-check.test.sh

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

test-windows-hooks:
needs: [select-tests-windows]
if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-hooks') }}
Expand Down Expand Up @@ -373,6 +413,14 @@ jobs:
if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/developer-experience/skills/setup/scripts/pointer-line.test.sh')
run: bash plugins/developer-experience/skills/setup/scripts/pointer-line.test.sh

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

test-windows-lib:
needs: [select-tests-windows]
if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-lib') }}
Expand Down Expand Up @@ -430,6 +478,14 @@ jobs:
$config.Output.Verbosity = 'Detailed'
Invoke-Pester -Configuration $config

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

test-windows-worktree:
needs: [select-tests-windows]
if: ${{ contains(fromJSON(needs.select-tests-windows.outputs.jobs || '[]'), 'test-windows-worktree') }}
Expand Down Expand Up @@ -477,6 +533,14 @@ jobs:
if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/planning/shared/tests/test_server.py')
run: python -m unittest discover -v -s plugins/planning/shared/tests -p 'test_tunnel.py'

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

# The planning shared-session suites on macOS, the third platform
# plugins/planning/shared/README.md names. detached.py's POSIX branches, file
# modes and the bash-run fake cloudflared run here as on Linux, but on Darwin:
Expand Down Expand Up @@ -539,3 +603,11 @@ jobs:
if: contains(fromJSON(needs.select-tests-windows.outputs.steps), 'plugins/planning/shared/tests/test_tunnel.py')
timeout-minutes: 3
run: python -m unittest discover -v -s plugins/planning/shared/tests -p 'test_tunnel.py'

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
16 changes: 16 additions & 0 deletions .github/workflows/release-publish-plugins.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,14 @@ jobs:
echo "Merged release pull request #$PR_NUMBER at $HEAD_SHA."
fi

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token

build:
needs: [merge]
if: ${{ !cancelled() && github.ref == 'refs/heads/main' && ((inputs.action || 'merge') != 'merge' || needs.merge.outputs.rebuild == 'true') }}
Expand Down Expand Up @@ -324,3 +332,11 @@ jobs:
printf '\n%s\n' "$line" >>"$body"
gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body"
fi

- name: Log the API budget at the end
if: always()
continue-on-error: true
timeout-minutes: 1
uses: ./.github/actions/log-api-budget # zizmor: ignore[self-repository]
with:
tokens: github-token
Loading