From 4e36f340cdb24b29196c3aeaf0f28bc57d2e4a7b Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Mon, 28 Sep 2026 11:10:18 +0000 Subject: [PATCH 1/7] fix: prevent nightly-promotion health check from misreporting a superseded E2E run as a failure The check-nightly-health job in weekly-nightly-promotion.yml resolved nightly's HEAD sha once before its 40-minute poll loop, then polled for required workflow runs against that pinned sha. nightly-build.yml's daily cron (0 9 * * *) and this workflow's own weekly cron shared the same 09:00 UTC slot (despite a comment claiming an offset), so nightly-build could re-sync nightly to a new sha mid-poll and dispatch its own E2E run for it. Because e2e-tests-split.yml's concurrency group is keyed on branch ref (not sha), that new dispatch cancelled the in-progress run for the old, still-tracked sha - which the poll loop then reported as a real failure and used to block promotion, even though current nightly HEAD was fully green (see #1404). Re-resolve nightly HEAD on every poll iteration; if it has advanced, follow the new sha and reset tracking instead of counting the superseded run's cancellation as a failure. Also move this workflow's cron to 09:30 UTC, genuinely offset from nightly-build's sync, as defense in depth against scheduler jitter observed to exceed 15 minutes. --- .../workflows/weekly-nightly-promotion.yml | 41 +++++++++++++++++-- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/.github/workflows/weekly-nightly-promotion.yml b/.github/workflows/weekly-nightly-promotion.yml index 1bd214441..e7b28a506 100644 --- a/.github/workflows/weekly-nightly-promotion.yml +++ b/.github/workflows/weekly-nightly-promotion.yml @@ -5,9 +5,13 @@ name: Nightly to Main Promotion on: schedule: - # Every Monday at 12:00 UTC (7:00am EST / 8:00am EDT) - # Offset from nightly sync (09:00 UTC) to avoid schedule race and allow validation completion. - - cron: '0 09 * * 1' + # Every Monday at 09:30 UTC - offset 30 minutes after nightly-build.yml's + # daily sync cron (0 9 * * *) so its branch sync + dispatch jobs have finished + # before we resolve nightly HEAD here. GH Actions scheduler jitter has been + # observed to delay cron firing by 15+ minutes (e.g. 2026-09-28: nightly-build's + # dispatch landed ~19 minutes after its nominal 09:00 UTC slot), so a flat + # "same minute" offset isn't enough margin; 30 minutes comfortably covers that. + - cron: '30 09 * * 1' workflow_dispatch: inputs: reason: @@ -75,7 +79,7 @@ jobs: repo: context.repo.repo, branch: 'nightly', }); - const nightlyHeadSha = nightlyBranch.commit.sha; + let nightlyHeadSha = nightlyBranch.commit.sha; core.info(`Current nightly HEAD: ${nightlyHeadSha}`); // Check critical workflows on the current nightly HEAD only. @@ -131,6 +135,35 @@ jobs: for (;;) { iteration += 1; + // Re-resolve nightly HEAD on every iteration. If it has advanced since + // we started (or since the last iteration), nightly moved forward mid-poll + // - e.g. nightly-build.yml's daily cron synced a new commit while we were + // still tracking the old one. In that case the old sha's in-flight run(s) + // may get cancelled purely because they were superseded (concurrency + // groups keyed on branch ref, not sha), not because anything is actually + // broken. Follow the new HEAD instead of letting that cancellation count + // as a failure. + const { data: currentNightlyBranch } = await github.rest.repos.getBranch({ + owner: context.repo.owner, + repo: context.repo.repo, + branch: 'nightly', + }); + const currentHeadSha = currentNightlyBranch.commit.sha; + + if (currentHeadSha !== nightlyHeadSha) { + core.info( + `Nightly HEAD advanced from ${nightlyHeadSha} to ${currentHeadSha} mid-poll ` + + `(iteration ${iteration}); resetting tracking to follow the new HEAD instead of ` + + 'treating the superseded sha\'s in-flight/cancelled runs as a failure.', + ); + nightlyHeadSha = currentHeadSha; + resolutions.clear(); + for (const workflow of criticalWorkflows) { + resolutions.set(workflow.workflowFile, { resolved: false, everFound: false }); + } + dispatched.clear(); + } + const { data: allRuns } = await github.rest.actions.listWorkflowRunsForRepo({ owner: context.repo.owner, repo: context.repo.repo, From d5206a6c3a8f45befebc5512569d02a24bb57877 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:13:08 +0000 Subject: [PATCH 2/7] chore(main): release 0.42.1 --- .release-please-manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 57953081c..cede7a525 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.42.0" + ".": "0.42.1" } From 927907bb6cb6126a9228e8fa3320eb7e71e01ff1 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Mon, 28 Sep 2026 13:18:58 +0000 Subject: [PATCH 3/7] fix: document release-please first-parent-adjacency skip and add recovery steps The 2026-09-28 weekly promotion merged real feat/fix commits (including the #1317 security hardening work) but release-please logged "No user facing commits found... skipping" because the previous release commit landed as the promotion merge's immediate first parent, starving its history walk before it could descend into nightly's second-parent commits. Documents the root cause and recovery in docs/troubleshooting/ and adds a pointer in CLAUDE.md's CI/CD section so it's recognized immediately next time, rather than re-diagnosed from scratch. --- CLAUDE.md | 1 + docs/troubleshooting/release-please-skips.md | 70 ++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 docs/troubleshooting/release-please-skips.md diff --git a/CLAUDE.md b/CLAUDE.md index d7f9a30bd..5f4eb5eeb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -145,6 +145,7 @@ never affected by this. - **Dependency bumps**: Renovate emits `deps:` (`.github/renovate.json` → `semanticCommitType: deps`, scope dropped). `deps:` is a release-triggering prefix for release-please, so a bump of anything that ships in the container cuts a patch release. CI-only tooling — GitHub Actions pins and the `custom.regex` trackers under `.github/workflows`, `scripts`, and `.github/skills` (golangci-lint, gotestsum, govulncheck, gopls, Syft, Grype, Semgrep image, CodeQL CLI, `NODE_VERSION`/`GO_VERSION`) — is forced back to `chore:` by packageRules so it doesn't cut a release. - **Beta**: `feature/beta-release` always builds. - **Weekly Promotion PRs** (`nightly → main`): ALWAYS merge using **"Create a merge commit"** — NEVER squash or rebase. Squash merging collapses all commits into bullet lines that the `auto-versioning` workflow cannot parse, silently preventing minor version bumps and producing empty release notes. +- **release-please silently skipping a release**: if a weekly promotion merges clean but no release PR appears, and the job log says `No user facing commits found since - skipping` — this is a known first-parent-adjacency quirk, not a missing prefix or config bug. See `docs/troubleshooting/release-please-skips.md` for root cause and recovery (land one ordinary hotfix commit on `main` to give the next run walk room, or force with `release-as`). - **History-Rewrite PRs**: If a PR touches files in `scripts/history-rewrite/` or `docs/plans/history_rewrite.md`, the PR description MUST include the history-rewrite checklist from `.github/PULL_REQUEST_TEMPLATE/history-rewrite.md`. ## Commit Slicing & PR Strategy diff --git a/docs/troubleshooting/release-please-skips.md b/docs/troubleshooting/release-please-skips.md new file mode 100644 index 000000000..96844ccdb --- /dev/null +++ b/docs/troubleshooting/release-please-skips.md @@ -0,0 +1,70 @@ +--- +title: release-please Silently Skipping a Release +description: Diagnosing and recovering when release-please logs "No user facing commits found" after a weekly nightly-to-main promotion despite real feat/fix commits being merged. +--- + +## release-please Silently Skipping a Release + +Symptom: the `release-please` GitHub Actions job runs green after a `Weekly: +Promote nightly to main` merge, but no release PR is opened, and the job log +ends with: + +```text +✔ Splitting 1 commits by path +❯ commits: 1 +✔ Considering: 1 commits +✔ No user facing commits found since - skipping +``` + +This happens even when the promotion genuinely brought in `feat:`/`fix:` +commits (rate-limit hardening, dependency fixes, etc.) — the commits are on +`main`, they are not yet in any tag, but release-please never saw them. + +### Root cause + +`release-please`'s commit walk on `main` stops as soon as it encounters the +previous release commit's SHA. The weekly promotion lands as a real two-parent +merge commit (per the "Create a merge commit" requirement in `CLAUDE.md` — +squash merges break the auto-versioning bullet parser). Its **first** parent +is whatever `main` pointed to before the merge, and its **second** parent is +the tip of `nightly`, which is where the actual `feat:`/`fix:` commits live. + +When nothing has landed directly on `main`'s first-parent line between one +release and the next weekly promotion, the previous release commit *is* the +promotion merge's immediate first parent. release-please's history walk +reaches that boundary SHA almost immediately and stops — before it has +paged far enough to descend into the merge's second-parent subtree, so it +never sees the real `nightly` commits at all, only the (non-conventional) +promotion merge subject itself. + +Contrast with a promotion where a couple of ordinary commits (a hotfix, a +`chore(main): release` commit, etc.) landed directly on `main` first between +releases: those extra first-parent hops give the walk enough room before it +hits the boundary, and it does surface buried `feat:`/`fix:` commits from the +second-parent side in the same run. This is a topology-adjacency quirk, not a +missing Conventional Commits prefix or a config problem in +`release-please-config.json` / `.release-please-manifest.json`. + +### How to confirm you're hitting this + +1. Pull the failed run's log: `gh run view --job --log`. + Look for `Set(1) { '' }` (the release boundary) landing as the + **second** item release-please backfills file lists for, right after the + promotion merge commit itself. +2. Verify real unreleased commits exist: + `git log --oneline ..origin/nightly`, and confirm they + are **not** ancestors of the last release commit: + `git merge-base --is-ancestor ` (exit non-zero + means genuinely new). + +### Recovery + +Land one ordinary commit directly on `main` via a hotfix branch + PR (per +the `CLAUDE.md` branching strategy — never push straight to `main`). This +gives the *next* release-please run one more first-parent hop of room before +it hits the (now-older) release boundary, which is normally enough for it to +also walk into the still-unreleased second-parent commits and open a correct +release PR. If a single spacer commit isn't enough (the buried commits are +very deep), force it instead: dispatch `release-please-action` manually with +an explicit `release-as` version to build the release PR directly, bypassing +the walk. From 6734782fb167bcb81e781bcc13c5d1a42a02f61d Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Mon, 28 Sep 2026 13:50:41 +0000 Subject: [PATCH 4/7] fix: correct root cause in release-please skip troubleshooting doc PR #1408 documented this as a first-parent-adjacency quirk, but that was wrong: a structurally identical prior promotion (2c59ac55) had the same non-conventional merge title and correctly surfaced a buried fix commit, and adding a spacer commit (per the original doc's recovery advice) did not fix the 2026-09-28 incident - it only reproduced the same truncation one commit later (see PR #1410). The real cause is date-ordering: release-please's commit walk stops at the release boundary's commit DATE, not a fixed graph position. Nightly commits older than an interim release that lands directly on main get silently stranded regardless of branch routing or spacer commits. Corrects the doc and adds the actual recovery procedure (hand-correct the release-please PR) used to fix v0.43.0. --- CLAUDE.md | 2 +- docs/troubleshooting/release-please-skips.md | 136 ++++++++++++------- 2 files changed, 90 insertions(+), 48 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 5f4eb5eeb..7764dcaec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -145,7 +145,7 @@ never affected by this. - **Dependency bumps**: Renovate emits `deps:` (`.github/renovate.json` → `semanticCommitType: deps`, scope dropped). `deps:` is a release-triggering prefix for release-please, so a bump of anything that ships in the container cuts a patch release. CI-only tooling — GitHub Actions pins and the `custom.regex` trackers under `.github/workflows`, `scripts`, and `.github/skills` (golangci-lint, gotestsum, govulncheck, gopls, Syft, Grype, Semgrep image, CodeQL CLI, `NODE_VERSION`/`GO_VERSION`) — is forced back to `chore:` by packageRules so it doesn't cut a release. - **Beta**: `feature/beta-release` always builds. - **Weekly Promotion PRs** (`nightly → main`): ALWAYS merge using **"Create a merge commit"** — NEVER squash or rebase. Squash merging collapses all commits into bullet lines that the `auto-versioning` workflow cannot parse, silently preventing minor version bumps and producing empty release notes. -- **release-please silently skipping a release**: if a weekly promotion merges clean but no release PR appears, and the job log says `No user facing commits found since - skipping` — this is a known first-parent-adjacency quirk, not a missing prefix or config bug. See `docs/troubleshooting/release-please-skips.md` for root cause and recovery (land one ordinary hotfix commit on `main` to give the next run walk room, or force with `release-as`). +- **release-please silently skipping or under-bumping a release**: if a weekly promotion merges clean but no release PR appears (or it proposes the wrong bump type), this is a known date-ordering quirk, not a missing prefix or config bug — release-please's commit walk stops once it reaches the release boundary's *commit date*, so nightly commits older than an interim release that landed directly on `main` get silently stranded. See `docs/troubleshooting/release-please-skips.md` for root cause and recovery (spacer commits do NOT fix this — hand-correct the release-please PR's manifest version and body directly). - **History-Rewrite PRs**: If a PR touches files in `scripts/history-rewrite/` or `docs/plans/history_rewrite.md`, the PR description MUST include the history-rewrite checklist from `.github/PULL_REQUEST_TEMPLATE/history-rewrite.md`. ## Commit Slicing & PR Strategy diff --git a/docs/troubleshooting/release-please-skips.md b/docs/troubleshooting/release-please-skips.md index 96844ccdb..f5d912a9f 100644 --- a/docs/troubleshooting/release-please-skips.md +++ b/docs/troubleshooting/release-please-skips.md @@ -1,13 +1,14 @@ --- -title: release-please Silently Skipping a Release -description: Diagnosing and recovering when release-please logs "No user facing commits found" after a weekly nightly-to-main promotion despite real feat/fix commits being merged. +title: release-please Silently Skipping or Under-Bumping a Release +description: Diagnosing and recovering when release-please misses real feat/fix commits after a weekly nightly-to-main promotion, either skipping the release entirely or cutting the wrong bump type. --- -## release-please Silently Skipping a Release +## release-please Silently Skipping or Under-Bumping a Release Symptom: the `release-please` GitHub Actions job runs green after a `Weekly: -Promote nightly to main` merge, but no release PR is opened, and the job log -ends with: +Promote nightly to main` merge, but either no release PR is opened, or the +release PR it opens proposes the wrong bump type (e.g. a patch bump when real +`feat:` commits were merged). The job log for the "skip" case ends with: ```text ✔ Splitting 1 commits by path @@ -17,54 +18,95 @@ ends with: ``` This happens even when the promotion genuinely brought in `feat:`/`fix:` -commits (rate-limit hardening, dependency fixes, etc.) — the commits are on -`main`, they are not yet in any tag, but release-please never saw them. +commits — the commits are on `main`, they are not yet in any tag, but +release-please never counted them. ### Root cause -`release-please`'s commit walk on `main` stops as soon as it encounters the -previous release commit's SHA. The weekly promotion lands as a real two-parent -merge commit (per the "Create a merge commit" requirement in `CLAUDE.md` — -squash merges break the auto-versioning bullet parser). Its **first** parent -is whatever `main` pointed to before the merge, and its **second** parent is -the tip of `nightly`, which is where the actual `feat:`/`fix:` commits live. - -When nothing has landed directly on `main`'s first-parent line between one -release and the next weekly promotion, the previous release commit *is* the -promotion merge's immediate first parent. release-please's history walk -reaches that boundary SHA almost immediately and stops — before it has -paged far enough to descend into the merge's second-parent subtree, so it -never sees the real `nightly` commits at all, only the (non-conventional) -promotion merge subject itself. - -Contrast with a promotion where a couple of ordinary commits (a hotfix, a -`chore(main): release` commit, etc.) landed directly on `main` first between -releases: those extra first-parent hops give the walk enough room before it -hits the boundary, and it does surface buried `feat:`/`fix:` commits from the -second-parent side in the same run. This is a topology-adjacency quirk, not a -missing Conventional Commits prefix or a config problem in -`release-please-config.json` / `.release-please-manifest.json`. +release-please's commit-collection walk effectively drops commits once it +reaches the previous release commit's SHA, and the commits it fetches from +GitHub come back ordered by **commit date**, not strict first-parent +topology. This means the walk stops the instant it reaches a commit at or +before the *release boundary's own commit date* — regardless of whether +older, structurally-unreleased commits exist elsewhere in the graph. + +The failure condition: a commit's own committer date is **older** than the +date of whatever commit currently sits as the release boundary on `main`, +at the moment it finally lands via a real (non-squash) merge. Concretely, +here is how it happened on 2026-09-28: + +1. `feat/auth-rate-limit-1317` merged into `nightly` on **2026-09-25**, + carrying real `feat:`/`fix(security):` commits dated that day. +2. `nightly` sat un-promoted for several days (weekly cadence). +3. Meanwhile, an unrelated fix landed **directly on `main`** and was + released as `v0.42.1` on the morning of **2026-09-28** — *before* that + week's nightly promotion had run. +4. When the weekly promotion merged later that same day, its second-parent + commits (the real `nightly` work) were now chronologically *older* than + `v0.42.1`'s own commit timestamp. release-please's date-ordered walk + reached the `v0.42.1` boundary before it ever paged far enough back to + see those older, still-unreleased commits, so it silently dropped them. + +This is **not** about branch routing — the same stranding can happen to a +long-lived branch merged directly into `main`, if it sits long enough that +an interim release lands first. It is **not** a missing Conventional +Commits prefix either: a structurally identical prior promotion (with an +equally non-conventional merge title) correctly picked up a buried `fix:` +commit, because in that case no interim release had jumped ahead of it. +And it is not fixed by landing spacer commits on `main` afterward — a +spacer only pushes the boundary *later*, widening the date gap rather than +closing it (this was tried on 2026-09-28 via PR #1408 and only produced a +9-commit-late `Set(1)` truncation instead of a full skip — see PR #1410). + +The two things that actually matter: + +1. **How long does non-squashed work sit before it lands on `main` via a + real merge?** The longer the gap, the bigger the window for an interim + release to jump ahead of it. +2. **Sequencing between releases and promotions.** If a same-week + `release-please` cut is allowed to land on `main` *before* that week's + pending nightly promotion, anything still waiting in `nightly` is at + risk of being dated older than the new boundary. ### How to confirm you're hitting this -1. Pull the failed run's log: `gh run view --job --log`. - Look for `Set(1) { '' }` (the release boundary) landing as the - **second** item release-please backfills file lists for, right after the - promotion merge commit itself. -2. Verify real unreleased commits exist: - `git log --oneline ..origin/nightly`, and confirm they - are **not** ancestors of the last release commit: - `git merge-base --is-ancestor ` (exit non-zero - means genuinely new). +1. Pull the run's log: `gh run view --job --log`. Look at + how many commits it reports considering (`commits: N`) versus how many + real conventional commits actually exist in the range. +2. Verify real unreleased commits exist and check their dates against the + boundary's date: + ``` + git log --format='%H|%cI|%s' ..origin/main | grep -E '\|(feat|fix|perf|revert)(\(|:)' + git log -1 --format='%cI' + ``` + Any matching commit with a committer date *older* than the boundary's is + at risk of being silently dropped, even though + `git merge-base --is-ancestor ` confirms it is + genuinely new (non-zero exit). ### Recovery -Land one ordinary commit directly on `main` via a hotfix branch + PR (per -the `CLAUDE.md` branching strategy — never push straight to `main`). This -gives the *next* release-please run one more first-parent hop of room before -it hits the (now-older) release boundary, which is normally enough for it to -also walk into the still-unreleased second-parent commits and open a correct -release PR. If a single spacer commit isn't enough (the buried commits are -very deep), force it instead: dispatch `release-please-action` manually with -an explicit `release-as` version to build the release PR directly, bypassing -the walk. +Do not rely on spacer commits or repeated re-runs — the walk is +deterministic on the current git graph and dates, so it will reproduce the +same result. Instead, hand-correct the release-please PR directly: + +1. Check out the `release-please--branches--main` branch. +2. Recompute the real commit set and correct bump type: + ``` + git log --format='%H|%s' ..origin/main | grep -E '^\w+\|(feat|fix|perf|revert)(\(|:)' + ``` + `feat:` present → minor bump (this repo has `bump-minor-pre-major: true`); + only `fix:`/`perf:` → patch. +3. Edit `.release-please-manifest.json` to the corrected version and push a + `fix:` commit to that branch. +4. Rewrite the PR body/release notes to list the real commits (release-please + only touches the manifest here since `skip-changelog: true` — there is no + `CHANGELOG.md` to reconcile), matching the existing bullet format: + `* ([](.../commit/))`, with `(#N)` references + in the subject converted to issue links and `:` rendered as + `**scope:**` prefix. +5. Merge the corrected PR. + +To prevent recurrence structurally, see the sequencing fix tracked for +`weekly-nightly-promotion.yml` / `release-please.yml` (ensure nightly always +promotes before an interim release is allowed to cut that week). From c8c144c334cbf71d2ac5132ad0b0095514d92096 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Mon, 28 Sep 2026 14:01:55 +0000 Subject: [PATCH 5/7] fix: guard release-please against cutting ahead of pending nightly promotion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit release-please's commit-collection walk on main stops at the previous release boundary's commit date rather than true git ancestry. When an interim hotfix release lands on main while nightly still has release-worthy commits dated older than that new boundary, those nightly commits are silently stranded once the promotion merge finally brings them in (see docs/troubleshooting/release-please-skips.md for the full 2026-09-28 incident writeup). Add a pre-check job to release-please.yml that compares nightly's tip against the commit that would become the new release boundary. If nightly has any feat/fix/perf/revert/deps commits older than that boundary, the release-please job is skipped for this push instead of cutting a release. This is self-healing: the guard re-evaluates on every push to main, so it naturally clears once nightly's backlog lands (whether via that week's promotion merge or a future one), at which point release-please resumes and correctly bundles everything that was deferred. The guard only defers when nightly genuinely has older, unpromoted release-worthy commits pending — an interim hotfix during a week where nightly has nothing pending, or where nightly's pending commits are newer than the boundary, is unaffected. Logic lives in scripts/ci/check-nightly-sequencing.sh with bats coverage in scripts/tests/check-nightly-sequencing.bats, wired into quality-checks.yml. --- .github/workflows/quality-checks.yml | 19 +++ .github/workflows/release-please.yml | 56 ++++++ scripts/ci/check-nightly-sequencing.sh | 116 +++++++++++++ scripts/tests/check-nightly-sequencing.bats | 178 ++++++++++++++++++++ 4 files changed, 369 insertions(+) create mode 100755 scripts/ci/check-nightly-sequencing.sh create mode 100644 scripts/tests/check-nightly-sequencing.bats diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 48583bfaf..88e88d73c 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -109,6 +109,25 @@ jobs: - name: bats run: bats scripts/tests/toolchain-key.bats scripts/tests/verify-toolchain-pin.bats + nightly-sequencing-guard-tests: + name: release-please nightly sequencing guard (bats) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install bats + shellcheck + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y bats shellcheck + + - name: shellcheck (shell severity=error) + run: | + shellcheck --severity=error scripts/ci/check-nightly-sequencing.sh + + - name: bats + run: bats scripts/tests/check-nightly-sequencing.bats + verify-toolchain-pin: name: Toolchain pin freshness (verify-toolchain-pin) runs-on: ubuntu-latest diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 3524195a7..b9fa2eaab 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -9,7 +9,42 @@ permissions: pull-requests: write jobs: + # Guards against the release-please date-ordering stranding bug (see + # docs/troubleshooting/release-please-skips.md): if this push were allowed + # to cut a release now, and `nightly` still has release-worthy commits + # dated older than this boundary waiting to be promoted, those commits + # would be silently dropped from every future release-please run once the + # promotion merge finally lands. Deferring here is self-healing — the next + # push to `main` (another interim commit, or the promotion merge itself) + # re-runs this check, and once nightly's backlog is caught up the range is + # empty and release-please resumes normally. + guard-nightly-sequencing: + name: Guard Nightly Promotion Sequencing + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + defer: ${{ steps.check.outputs.defer }} + reason: ${{ steps.check.outputs.reason }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Fetch nightly + run: git fetch origin nightly:refs/remotes/origin/nightly || true + + - name: Check nightly sequencing + id: check + env: + TARGET_REF: ${{ github.sha }} + SOURCE_REF: origin/nightly + run: bash scripts/ci/check-nightly-sequencing.sh + release-please: + name: Release Please + needs: guard-nightly-sequencing + if: needs.guard-nightly-sequencing.outputs.defer != 'true' runs-on: ubuntu-latest steps: - uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5 @@ -17,3 +52,24 @@ jobs: config-file: release-please-config.json manifest-file: .release-please-manifest.json target-branch: main + + deferred: + name: Release Deferred (Pending Nightly Promotion) + needs: guard-nightly-sequencing + if: needs.guard-nightly-sequencing.outputs.defer == 'true' + runs-on: ubuntu-latest + steps: + - name: Log defer reason + env: + REASON: ${{ needs.guard-nightly-sequencing.outputs.reason }} + run: | + { + echo "## release-please deferred" + echo "" + echo "This run was skipped to avoid landing a release boundary ahead of" + echo "a pending nightly promotion. It will run automatically on the next" + echo "push to \`main\` — typically this week's nightly promotion merge." + echo "" + echo "**Reason:** ${REASON}" + } >> "$GITHUB_STEP_SUMMARY" + echo "::notice title=release-please deferred::${REASON}" diff --git a/scripts/ci/check-nightly-sequencing.sh b/scripts/ci/check-nightly-sequencing.sh new file mode 100755 index 000000000..a54bc3969 --- /dev/null +++ b/scripts/ci/check-nightly-sequencing.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Guards against the release-please date-ordering stranding bug documented in +# docs/troubleshooting/release-please-skips.md. +# +# Root cause (confirmed 2026-09-28, PR #1411): release-please's commit- +# collection walk on `main` stops once it reaches the previous release +# boundary's *commit date*, not true git ancestry. If a same-week +# release-please cut lands on `main` (creating a new boundary dated "now") +# while `nightly` still has release-worthy commits dated *older* than that +# boundary waiting to be promoted, those commits are silently dropped from +# every future release-please run once the promotion merge finally lands. +# +# This script decides whether the release-please job in +# .github/workflows/release-please.yml should run (defer=false) or be +# skipped for this push (defer=true) to avoid creating that boundary early. +# It is self-healing: the next push to `main` (another interim commit, or +# the eventual nightly promotion merge) re-runs this check. Once nightly's +# commits are ancestors of TARGET_REF, the range is empty and release-please +# resumes normally, correctly bundling everything that was deferred. +# +# Usage: +# TARGET_REF= SOURCE_REF= scripts/ci/check-nightly-sequencing.sh +# +# Inputs (env vars, all optional): +# TARGET_REF Commit that would become the new release boundary +# if release-please proceeds. Default: HEAD. +# SOURCE_REF Tip of the branch that owns pending promotion work. +# Default: origin/nightly. +# COMMIT_TYPE_PATTERN Extended-regex subject-line pattern for commit +# types release-please treats as release-worthy in +# this repo (see CLAUDE.md's CI/CD & Commit +# Conventions section: feat/fix/perf trigger builds, +# deps: is release-triggering via Renovate, revert is +# a standard Conventional Commits release type). +# Default matches feat/fix/perf/revert/deps, with +# optional (scope) and breaking-change `!`. +# +# Output: two `key=value` lines on stdout (also appended to $GITHUB_OUTPUT +# when set, so this can be used directly as a GitHub Actions step): +# defer=true|false +# reason= +# +# Exit codes: 0 for any successful decision (including defer=true — that is +# a valid decision, not a script failure). Non-zero only on genuine usage +# errors (e.g. TARGET_REF does not resolve). + +set -euo pipefail + +TARGET_REF="${TARGET_REF:-HEAD}" +SOURCE_REF="${SOURCE_REF:-origin/nightly}" +COMMIT_TYPE_PATTERN="${COMMIT_TYPE_PATTERN:-^(feat|fix|perf|revert|deps)(\([^)]*\))?!?:}" +DOCS_LINK="docs/troubleshooting/release-please-skips.md" + +emit() { + local defer="$1" + local reason="$2" + echo "defer=${defer}" + echo "reason=${reason}" + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + { + echo "defer=${defer}" + echo "reason=${reason}" + } >>"${GITHUB_OUTPUT}" + fi +} + +if ! git rev-parse --verify -q "${TARGET_REF}^{commit}" >/dev/null; then + echo "check-nightly-sequencing: TARGET_REF '${TARGET_REF}' does not resolve to a commit" >&2 + exit 2 +fi + +if ! git rev-parse --verify -q "${SOURCE_REF}^{commit}" >/dev/null 2>&1; then + emit "false" "SOURCE_REF '${SOURCE_REF}' does not resolve (branch missing or not fetched); nothing to guard against" + exit 0 +fi + +target_epoch="$(git log -1 --format=%ct "${TARGET_REF}")" + +# Commits reachable from SOURCE_REF but not TARGET_REF: nightly's unpromoted +# backlog relative to the boundary this release-please run would create. +# --no-merges excludes sync/promotion merge commits themselves; we only care +# about the real conventional commits they carry. +matches=() +while IFS=$'\x1f' read -r sha epoch iso subject; do + [[ -z "${sha}" ]] && continue + if [[ "${subject}" =~ ${COMMIT_TYPE_PATTERN} ]]; then + matches+=("${sha}"$'\x1f'"${epoch}"$'\x1f'"${iso}"$'\x1f'"${subject}") + fi +done < <(git log --no-merges --format='%H%x1f%ct%x1f%cI%x1f%s' "${TARGET_REF}..${SOURCE_REF}" 2>/dev/null || true) + +if [[ "${#matches[@]}" -eq 0 ]]; then + emit "false" "nightly has no release-worthy commits ahead of this boundary; safe to release" + exit 0 +fi + +# Find the oldest qualifying commit (lowest committer-date epoch). +oldest="" +oldest_epoch="" +at_risk_count=0 +for entry in "${matches[@]}"; do + IFS=$'\x1f' read -r sha epoch iso subject <<<"${entry}" + if [[ "${epoch}" -lt "${target_epoch}" ]]; then + at_risk_count=$((at_risk_count + 1)) + if [[ -z "${oldest_epoch}" || "${epoch}" -lt "${oldest_epoch}" ]]; then + oldest_epoch="${epoch}" + oldest="${sha:0:7} @ ${iso} - ${subject}" + fi + fi +done + +if [[ "${at_risk_count}" -eq 0 ]]; then + emit "false" "nightly has ${#matches[@]} release-worthy commit(s) ahead, but none predate this boundary; safe to release" + exit 0 +fi + +emit "true" "nightly has ${at_risk_count} unpromoted release-worthy commit(s) older than this release boundary (oldest: ${oldest}); deferring to avoid the release-please date-ordering stranding bug, see ${DOCS_LINK}" diff --git a/scripts/tests/check-nightly-sequencing.bats b/scripts/tests/check-nightly-sequencing.bats new file mode 100644 index 000000000..b5643fb96 --- /dev/null +++ b/scripts/tests/check-nightly-sequencing.bats @@ -0,0 +1,178 @@ +#!/usr/bin/env bats +# +# Covers scripts/ci/check-nightly-sequencing.sh, the guard that prevents a +# release-please cut from landing on `main` while `nightly` still has +# unpromoted release-worthy commits dated older than the incoming release +# boundary (the date-ordering stranding bug documented in +# docs/troubleshooting/release-please-skips.md). +# +# Each test builds an isolated fake git repo so history/dates are fully +# controlled, following the pattern used by +# scripts/tests/local-patch-report_baseline.bats. + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + SCRIPT_UNDER_TEST="$REPO_ROOT/scripts/ci/check-nightly-sequencing.sh" + + TMPROOT="$(mktemp -d)" + git -C "$TMPROOT" init -q -b main + git -C "$TMPROOT" config user.email "test@example.com" + git -C "$TMPROOT" config user.name "Test Runner" + + unset TARGET_REF SOURCE_REF GITHUB_OUTPUT +} + +teardown() { + rm -rf "$TMPROOT" +} + +# Commits with an explicit, deterministic author+committer date so ordering +# never depends on when the test happens to run. +commit_at() { + local date="$1" message="$2" + GIT_AUTHOR_DATE="$date" GIT_COMMITTER_DATE="$date" \ + git -C "$TMPROOT" commit -q --allow-empty -m "$message" +} + +run_check() { + (cd "$TMPROOT" && "$SCRIPT_UNDER_TEST") +} + +@test "no divergence between target and source: defer=false" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + + export TARGET_REF="main" SOURCE_REF="nightly" + run run_check + [ "$status" -eq 0 ] + [[ "$output" == *"defer=false"* ]] + [[ "$output" == *"no release-worthy commits ahead"* ]] +} + +@test "nightly ahead with only non-release-worthy commits: defer=false" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + git -C "$TMPROOT" checkout -q nightly + commit_at "2026-09-21T09:00:00" "docs: update README" + commit_at "2026-09-22T09:00:00" "chore: tidy imports" + + export TARGET_REF="main" SOURCE_REF="nightly" + run run_check + [ "$status" -eq 0 ] + [[ "$output" == *"defer=false"* ]] + [[ "$output" == *"no release-worthy commits ahead"* ]] +} + +@test "nightly has release-worthy commits ahead, but all newer than the boundary: defer=false" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + commit_at "2026-09-21T09:00:00" "fix: hotfix on main" + git -C "$TMPROOT" checkout -q nightly + commit_at "2026-09-25T10:00:00" "fix: newer nightly fix" + + export TARGET_REF="main" SOURCE_REF="nightly" + run run_check + [ "$status" -eq 0 ] + [[ "$output" == *"defer=false"* ]] + [[ "$output" == *"none predate this boundary"* ]] +} + +@test "nightly has an older unpromoted fix commit: defer=true, names oldest" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + git -C "$TMPROOT" checkout -q nightly + commit_at "2026-09-25T10:00:00" "fix(security): harden input validation in the API layer" + git -C "$TMPROOT" checkout -q main + commit_at "2026-09-28T11:44:00" "fix: unrelated hotfix" + + export TARGET_REF="main" SOURCE_REF="nightly" + run run_check + [ "$status" -eq 0 ] + [[ "$output" == *"defer=true"* ]] + [[ "$output" == *"1 unpromoted release-worthy commit"* ]] + [[ "$output" == *"fix(security): harden input validation in the API layer"* ]] + [[ "$output" == *"docs/troubleshooting/release-please-skips.md"* ]] +} + +@test "multiple at-risk commits: count is correct and oldest (not newest) is reported" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + git -C "$TMPROOT" checkout -q nightly + commit_at "2026-09-23T09:00:00" "feat: add newer feature" + commit_at "2026-09-21T09:00:00" "fix: oldest buried fix" + git -C "$TMPROOT" checkout -q main + commit_at "2026-09-28T11:44:00" "fix: unrelated hotfix" + + export TARGET_REF="main" SOURCE_REF="nightly" + run run_check + [ "$status" -eq 0 ] + [[ "$output" == *"defer=true"* ]] + [[ "$output" == *"2 unpromoted release-worthy commit"* ]] + [[ "$output" == *"oldest buried fix"* ]] +} + +@test "breaking-change and scoped subjects are recognized" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + git -C "$TMPROOT" checkout -q nightly + commit_at "2026-09-21T09:00:00" "fix(api)!: breaking change to auth header" + git -C "$TMPROOT" checkout -q main + commit_at "2026-09-28T11:44:00" "fix: unrelated hotfix" + + export TARGET_REF="main" SOURCE_REF="nightly" + run run_check + [[ "$output" == *"defer=true"* ]] + [[ "$output" == *"1 unpromoted release-worthy commit"* ]] +} + +@test "deps: commits are treated as release-worthy" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + git -C "$TMPROOT" checkout -q nightly + commit_at "2026-09-21T09:00:00" "deps: bump some-lib to 2.0.0" + git -C "$TMPROOT" checkout -q main + commit_at "2026-09-28T11:44:00" "fix: unrelated hotfix" + + export TARGET_REF="main" SOURCE_REF="nightly" + run run_check + [[ "$output" == *"defer=true"* ]] +} + +@test "missing SOURCE_REF (nightly branch not fetched): defer=false, does not fail" { + commit_at "2026-09-20T09:00:00" "chore: init" + + export TARGET_REF="main" SOURCE_REF="origin/nightly-does-not-exist" + run run_check + [ "$status" -eq 0 ] + [[ "$output" == *"defer=false"* ]] + [[ "$output" == *"does not resolve"* ]] +} + +@test "invalid TARGET_REF is a hard usage error, not a silent defer=false" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + + export TARGET_REF="not-a-real-ref" SOURCE_REF="nightly" + run run_check + [ "$status" -eq 2 ] + [[ "$output" != *"defer="* ]] +} + +@test "writes defer and reason to GITHUB_OUTPUT when set" { + commit_at "2026-09-20T09:00:00" "chore: init" + git -C "$TMPROOT" branch nightly + git -C "$TMPROOT" checkout -q nightly + commit_at "2026-09-25T10:00:00" "fix: buried fix" + git -C "$TMPROOT" checkout -q main + commit_at "2026-09-28T11:44:00" "fix: unrelated hotfix" + + OUT_FILE="$TMPROOT/gh_output" + : >"$OUT_FILE" + export TARGET_REF="main" SOURCE_REF="nightly" GITHUB_OUTPUT="$OUT_FILE" + run run_check + [ "$status" -eq 0 ] + + run cat "$OUT_FILE" + [[ "$output" == *"defer=true"* ]] + [[ "$output" == *"reason="* ]] +} From 3fdc992f978d5ac937f7a10c90093c1c340aeaf2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 14:20:33 +0000 Subject: [PATCH 6/7] chore(main): release 0.42.2 --- .release-please-manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index cede7a525..a623f136f 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.42.1" + ".": "0.42.2" } From 39b2d3765c41c9a5a44beb1b9259a4064ef26871 Mon Sep 17 00:00:00 2001 From: Jeremy Hatfield Date: Mon, 28 Sep 2026 14:23:27 +0000 Subject: [PATCH 7/7] fix: correct release-please manifest to 0.43.0 (minor), again release-please's own automated run reset this branch back to 0.42.2 when PRs #1411 and #1412 merged (each push to main re-triggers release-please.yml, which recomputes and force-overwrites this PR from scratch). The underlying date-ordering bug (see docs/troubleshooting/release-please-skips.md) still applies to the gap between the v0.42.1 tag and current main, so the recomputed PR is wrong for the same reason as before. Re-applying the correction; merging this immediately to avoid another reset from a subsequent push. --- .release-please-manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index a623f136f..2ff8218f3 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.42.2" + ".": "0.43.0" }