Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions .github/workflows/quality-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,25 @@ jobs:
- name: bats
run: bats scripts/tests/toolchain-key.bats scripts/tests/verify-toolchain-pin.bats

nightly-sequencing-guard-tests:
name: release-please nightly sequencing guard (bats)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

- name: Install bats + shellcheck
run: |
set -euo pipefail
sudo apt-get update
sudo apt-get install -y bats shellcheck

- name: shellcheck (shell severity=error)
run: |
shellcheck --severity=error scripts/ci/check-nightly-sequencing.sh

- name: bats
run: bats scripts/tests/check-nightly-sequencing.bats

verify-toolchain-pin:
name: Toolchain pin freshness (verify-toolchain-pin)
runs-on: ubuntu-latest
Expand Down
56 changes: 56 additions & 0 deletions .github/workflows/release-please.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,67 @@ permissions:
pull-requests: write

jobs:
# Guards against the release-please date-ordering stranding bug (see
# docs/troubleshooting/release-please-skips.md): if this push were allowed
# to cut a release now, and `nightly` still has release-worthy commits
# dated older than this boundary waiting to be promoted, those commits
# would be silently dropped from every future release-please run once the
# promotion merge finally lands. Deferring here is self-healing — the next
# push to `main` (another interim commit, or the promotion merge itself)
# re-runs this check, and once nightly's backlog is caught up the range is
# empty and release-please resumes normally.
guard-nightly-sequencing:
name: Guard Nightly Promotion Sequencing
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
defer: ${{ steps.check.outputs.defer }}
reason: ${{ steps.check.outputs.reason }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0

- name: Fetch nightly
run: git fetch origin nightly:refs/remotes/origin/nightly || true

- name: Check nightly sequencing
id: check
env:
TARGET_REF: ${{ github.sha }}
SOURCE_REF: origin/nightly
run: bash scripts/ci/check-nightly-sequencing.sh

release-please:
name: Release Please
needs: guard-nightly-sequencing
if: needs.guard-nightly-sequencing.outputs.defer != 'true'
runs-on: ubuntu-latest
steps:
- uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5
with:
config-file: release-please-config.json
manifest-file: .release-please-manifest.json
target-branch: main

deferred:
name: Release Deferred (Pending Nightly Promotion)
needs: guard-nightly-sequencing
if: needs.guard-nightly-sequencing.outputs.defer == 'true'
runs-on: ubuntu-latest
steps:
- name: Log defer reason
env:
REASON: ${{ needs.guard-nightly-sequencing.outputs.reason }}
run: |
{
echo "## release-please deferred"
echo ""
echo "This run was skipped to avoid landing a release boundary ahead of"
echo "a pending nightly promotion. It will run automatically on the next"
echo "push to \`main\` — typically this week's nightly promotion merge."
echo ""
echo "**Reason:** ${REASON}"
} >> "$GITHUB_STEP_SUMMARY"
echo "::notice title=release-please deferred::${REASON}"
41 changes: 37 additions & 4 deletions .github/workflows/weekly-nightly-promotion.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@ name: Nightly to Main Promotion

on:
schedule:
# Every Monday at 12:00 UTC (7:00am EST / 8:00am EDT)
# Offset from nightly sync (09:00 UTC) to avoid schedule race and allow validation completion.
- cron: '0 09 * * 1'
# Every Monday at 09:30 UTC - offset 30 minutes after nightly-build.yml's
# daily sync cron (0 9 * * *) so its branch sync + dispatch jobs have finished
# before we resolve nightly HEAD here. GH Actions scheduler jitter has been
# observed to delay cron firing by 15+ minutes (e.g. 2026-09-28: nightly-build's
# dispatch landed ~19 minutes after its nominal 09:00 UTC slot), so a flat
# "same minute" offset isn't enough margin; 30 minutes comfortably covers that.
- cron: '30 09 * * 1'
workflow_dispatch:
inputs:
reason:
Expand Down Expand Up @@ -75,7 +79,7 @@ jobs:
repo: context.repo.repo,
branch: 'nightly',
});
const nightlyHeadSha = nightlyBranch.commit.sha;
let nightlyHeadSha = nightlyBranch.commit.sha;
core.info(`Current nightly HEAD: ${nightlyHeadSha}`);

// Check critical workflows on the current nightly HEAD only.
Expand Down Expand Up @@ -131,6 +135,35 @@ jobs:
for (;;) {
iteration += 1;

// Re-resolve nightly HEAD on every iteration. If it has advanced since
// we started (or since the last iteration), nightly moved forward mid-poll
// - e.g. nightly-build.yml's daily cron synced a new commit while we were
// still tracking the old one. In that case the old sha's in-flight run(s)
// may get cancelled purely because they were superseded (concurrency
// groups keyed on branch ref, not sha), not because anything is actually
// broken. Follow the new HEAD instead of letting that cancellation count
// as a failure.
const { data: currentNightlyBranch } = await github.rest.repos.getBranch({
owner: context.repo.owner,
repo: context.repo.repo,
branch: 'nightly',
});
const currentHeadSha = currentNightlyBranch.commit.sha;

if (currentHeadSha !== nightlyHeadSha) {
core.info(
`Nightly HEAD advanced from ${nightlyHeadSha} to ${currentHeadSha} mid-poll ` +
`(iteration ${iteration}); resetting tracking to follow the new HEAD instead of ` +
'treating the superseded sha\'s in-flight/cancelled runs as a failure.',
);
nightlyHeadSha = currentHeadSha;
resolutions.clear();
for (const workflow of criticalWorkflows) {
resolutions.set(workflow.workflowFile, { resolved: false, everFound: false });
}
dispatched.clear();
}

const { data: allRuns } = await github.rest.actions.listWorkflowRunsForRepo({
owner: context.repo.owner,
repo: context.repo.repo,
Expand Down
2 changes: 1 addition & 1 deletion .release-please-manifest.json
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{
".": "0.42.0"
".": "0.43.0"
}
1 change: 1 addition & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ never affected by this.
- **Dependency bumps**: Renovate emits `deps:` (`.github/renovate.json` → `semanticCommitType: deps`, scope dropped). `deps:` is a release-triggering prefix for release-please, so a bump of anything that ships in the container cuts a patch release. CI-only tooling — GitHub Actions pins and the `custom.regex` trackers under `.github/workflows`, `scripts`, and `.github/skills` (golangci-lint, gotestsum, govulncheck, gopls, Syft, Grype, Semgrep image, CodeQL CLI, `NODE_VERSION`/`GO_VERSION`) — is forced back to `chore:` by packageRules so it doesn't cut a release.
- **Beta**: `feature/beta-release` always builds.
- **Weekly Promotion PRs** (`nightly → main`): ALWAYS merge using **"Create a merge commit"** — NEVER squash or rebase. Squash merging collapses all commits into bullet lines that the `auto-versioning` workflow cannot parse, silently preventing minor version bumps and producing empty release notes.
- **release-please silently skipping or under-bumping a release**: if a weekly promotion merges clean but no release PR appears (or it proposes the wrong bump type), this is a known date-ordering quirk, not a missing prefix or config bug — release-please's commit walk stops once it reaches the release boundary's *commit date*, so nightly commits older than an interim release that landed directly on `main` get silently stranded. See `docs/troubleshooting/release-please-skips.md` for root cause and recovery (spacer commits do NOT fix this — hand-correct the release-please PR's manifest version and body directly).
- **History-Rewrite PRs**: If a PR touches files in `scripts/history-rewrite/` or `docs/plans/history_rewrite.md`, the PR description MUST include the history-rewrite checklist from `.github/PULL_REQUEST_TEMPLATE/history-rewrite.md`.

## Commit Slicing & PR Strategy
Expand Down
112 changes: 112 additions & 0 deletions docs/troubleshooting/release-please-skips.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
---
title: release-please Silently Skipping or Under-Bumping a Release
description: Diagnosing and recovering when release-please misses real feat/fix commits after a weekly nightly-to-main promotion, either skipping the release entirely or cutting the wrong bump type.
---

## release-please Silently Skipping or Under-Bumping a Release

Symptom: the `release-please` GitHub Actions job runs green after a `Weekly:
Promote nightly to main` merge, but either no release PR is opened, or the
release PR it opens proposes the wrong bump type (e.g. a patch bump when real
`feat:` commits were merged). The job log for the "skip" case ends with:

```text
✔ Splitting 1 commits by path
❯ commits: 1
✔ Considering: 1 commits
✔ No user facing commits found since <last-release-sha> - skipping
```

This happens even when the promotion genuinely brought in `feat:`/`fix:`
commits — the commits are on `main`, they are not yet in any tag, but
release-please never counted them.

### Root cause

release-please's commit-collection walk effectively drops commits once it
reaches the previous release commit's SHA, and the commits it fetches from
GitHub come back ordered by **commit date**, not strict first-parent
topology. This means the walk stops the instant it reaches a commit at or
before the *release boundary's own commit date* — regardless of whether
older, structurally-unreleased commits exist elsewhere in the graph.

The failure condition: a commit's own committer date is **older** than the
date of whatever commit currently sits as the release boundary on `main`,
at the moment it finally lands via a real (non-squash) merge. Concretely,
here is how it happened on 2026-09-28:

1. `feat/auth-rate-limit-1317` merged into `nightly` on **2026-09-25**,
carrying real `feat:`/`fix(security):` commits dated that day.
2. `nightly` sat un-promoted for several days (weekly cadence).
3. Meanwhile, an unrelated fix landed **directly on `main`** and was
released as `v0.42.1` on the morning of **2026-09-28** — *before* that
week's nightly promotion had run.
4. When the weekly promotion merged later that same day, its second-parent
commits (the real `nightly` work) were now chronologically *older* than
`v0.42.1`'s own commit timestamp. release-please's date-ordered walk
reached the `v0.42.1` boundary before it ever paged far enough back to
see those older, still-unreleased commits, so it silently dropped them.

This is **not** about branch routing — the same stranding can happen to a
long-lived branch merged directly into `main`, if it sits long enough that
an interim release lands first. It is **not** a missing Conventional
Commits prefix either: a structurally identical prior promotion (with an
equally non-conventional merge title) correctly picked up a buried `fix:`
commit, because in that case no interim release had jumped ahead of it.
And it is not fixed by landing spacer commits on `main` afterward — a
spacer only pushes the boundary *later*, widening the date gap rather than
closing it (this was tried on 2026-09-28 via PR #1408 and only produced a
9-commit-late `Set(1)` truncation instead of a full skip — see PR #1410).

The two things that actually matter:

1. **How long does non-squashed work sit before it lands on `main` via a
real merge?** The longer the gap, the bigger the window for an interim
release to jump ahead of it.
2. **Sequencing between releases and promotions.** If a same-week
`release-please` cut is allowed to land on `main` *before* that week's
pending nightly promotion, anything still waiting in `nightly` is at
risk of being dated older than the new boundary.

### How to confirm you're hitting this

1. Pull the run's log: `gh run view <run-id> --job <job-id> --log`. Look at
how many commits it reports considering (`commits: N`) versus how many
real conventional commits actually exist in the range.
2. Verify real unreleased commits exist and check their dates against the
boundary's date:
```
git log --format='%H|%cI|%s' <last-release-sha>..origin/main | grep -E '\|(feat|fix|perf|revert)(\(|:)'
git log -1 --format='%cI' <last-release-sha>
```
Any matching commit with a committer date *older* than the boundary's is
at risk of being silently dropped, even though
`git merge-base --is-ancestor <sha> <last-release-sha>` confirms it is
genuinely new (non-zero exit).

### Recovery

Do not rely on spacer commits or repeated re-runs — the walk is
deterministic on the current git graph and dates, so it will reproduce the
same result. Instead, hand-correct the release-please PR directly:

1. Check out the `release-please--branches--main` branch.
2. Recompute the real commit set and correct bump type:
```
git log --format='%H|%s' <last-release-sha>..origin/main | grep -E '^\w+\|(feat|fix|perf|revert)(\(|:)'
```
`feat:` present → minor bump (this repo has `bump-minor-pre-major: true`);
only `fix:`/`perf:` → patch.
3. Edit `.release-please-manifest.json` to the corrected version and push a
`fix:` commit to that branch.
4. Rewrite the PR body/release notes to list the real commits (release-please
only touches the manifest here since `skip-changelog: true` — there is no
`CHANGELOG.md` to reconcile), matching the existing bullet format:
`* <description> ([<short-sha>](.../commit/<sha>))`, with `(#N)` references
in the subject converted to issue links and `<scope>:` rendered as
`**scope:**` prefix.
5. Merge the corrected PR.

To prevent recurrence structurally, see the sequencing fix tracked for
`weekly-nightly-promotion.yml` / `release-please.yml` (ensure nightly always
promotes before an interim release is allowed to cut that week).
116 changes: 116 additions & 0 deletions scripts/ci/check-nightly-sequencing.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
#!/usr/bin/env bash
# Guards against the release-please date-ordering stranding bug documented in
# docs/troubleshooting/release-please-skips.md.
#
# Root cause (confirmed 2026-09-28, PR #1411): release-please's commit-
# collection walk on `main` stops once it reaches the previous release
# boundary's *commit date*, not true git ancestry. If a same-week
# release-please cut lands on `main` (creating a new boundary dated "now")
# while `nightly` still has release-worthy commits dated *older* than that
# boundary waiting to be promoted, those commits are silently dropped from
# every future release-please run once the promotion merge finally lands.
#
# This script decides whether the release-please job in
# .github/workflows/release-please.yml should run (defer=false) or be
# skipped for this push (defer=true) to avoid creating that boundary early.
# It is self-healing: the next push to `main` (another interim commit, or
# the eventual nightly promotion merge) re-runs this check. Once nightly's
# commits are ancestors of TARGET_REF, the range is empty and release-please
# resumes normally, correctly bundling everything that was deferred.
#
# Usage:
# TARGET_REF=<sha-or-ref> SOURCE_REF=<sha-or-ref> scripts/ci/check-nightly-sequencing.sh
#
# Inputs (env vars, all optional):
# TARGET_REF Commit that would become the new release boundary
# if release-please proceeds. Default: HEAD.
# SOURCE_REF Tip of the branch that owns pending promotion work.
# Default: origin/nightly.
# COMMIT_TYPE_PATTERN Extended-regex subject-line pattern for commit
# types release-please treats as release-worthy in
# this repo (see CLAUDE.md's CI/CD & Commit
# Conventions section: feat/fix/perf trigger builds,
# deps: is release-triggering via Renovate, revert is
# a standard Conventional Commits release type).
# Default matches feat/fix/perf/revert/deps, with
# optional (scope) and breaking-change `!`.
#
# Output: two `key=value` lines on stdout (also appended to $GITHUB_OUTPUT
# when set, so this can be used directly as a GitHub Actions step):
# defer=true|false
# reason=<single-line human-readable explanation>
#
# Exit codes: 0 for any successful decision (including defer=true — that is
# a valid decision, not a script failure). Non-zero only on genuine usage
# errors (e.g. TARGET_REF does not resolve).

set -euo pipefail

TARGET_REF="${TARGET_REF:-HEAD}"
SOURCE_REF="${SOURCE_REF:-origin/nightly}"
COMMIT_TYPE_PATTERN="${COMMIT_TYPE_PATTERN:-^(feat|fix|perf|revert|deps)(\([^)]*\))?!?:}"
DOCS_LINK="docs/troubleshooting/release-please-skips.md"

emit() {
local defer="$1"
local reason="$2"
echo "defer=${defer}"
echo "reason=${reason}"
if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
{
echo "defer=${defer}"
echo "reason=${reason}"
} >>"${GITHUB_OUTPUT}"
fi
}

if ! git rev-parse --verify -q "${TARGET_REF}^{commit}" >/dev/null; then
echo "check-nightly-sequencing: TARGET_REF '${TARGET_REF}' does not resolve to a commit" >&2
exit 2
fi

if ! git rev-parse --verify -q "${SOURCE_REF}^{commit}" >/dev/null 2>&1; then
emit "false" "SOURCE_REF '${SOURCE_REF}' does not resolve (branch missing or not fetched); nothing to guard against"
exit 0
fi

target_epoch="$(git log -1 --format=%ct "${TARGET_REF}")"

# Commits reachable from SOURCE_REF but not TARGET_REF: nightly's unpromoted
# backlog relative to the boundary this release-please run would create.
# --no-merges excludes sync/promotion merge commits themselves; we only care
# about the real conventional commits they carry.
matches=()
while IFS=$'\x1f' read -r sha epoch iso subject; do
[[ -z "${sha}" ]] && continue
if [[ "${subject}" =~ ${COMMIT_TYPE_PATTERN} ]]; then
matches+=("${sha}"$'\x1f'"${epoch}"$'\x1f'"${iso}"$'\x1f'"${subject}")
fi
done < <(git log --no-merges --format='%H%x1f%ct%x1f%cI%x1f%s' "${TARGET_REF}..${SOURCE_REF}" 2>/dev/null || true)

if [[ "${#matches[@]}" -eq 0 ]]; then
emit "false" "nightly has no release-worthy commits ahead of this boundary; safe to release"
exit 0
fi

# Find the oldest qualifying commit (lowest committer-date epoch).
oldest=""
oldest_epoch=""
at_risk_count=0
for entry in "${matches[@]}"; do
IFS=$'\x1f' read -r sha epoch iso subject <<<"${entry}"
if [[ "${epoch}" -lt "${target_epoch}" ]]; then
at_risk_count=$((at_risk_count + 1))
if [[ -z "${oldest_epoch}" || "${epoch}" -lt "${oldest_epoch}" ]]; then
oldest_epoch="${epoch}"
oldest="${sha:0:7} @ ${iso} - ${subject}"
fi
fi
done

if [[ "${at_risk_count}" -eq 0 ]]; then
emit "false" "nightly has ${#matches[@]} release-worthy commit(s) ahead, but none predate this boundary; safe to release"
exit 0
fi

emit "true" "nightly has ${at_risk_count} unpromoted release-worthy commit(s) older than this release boundary (oldest: ${oldest}); deferring to avoid the release-please date-ordering stranding bug, see ${DOCS_LINK}"
Loading
Loading