Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
141 changes: 141 additions & 0 deletions .github/workflows/action-tag-recovery.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
name: Recover major Action tag

on:
workflow_dispatch:
inputs:
tag:
description: Existing successful stable lockstep release tag to restore (vMAJOR.MINOR.PATCH).
required: true
type: string

concurrency:
# This is intentionally identical to release.yml: forward promotion and
# recovery must never race each other.
group: release-${{ github.repository }}
cancel-in-progress: false

jobs:
validate-context:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Require the canonical repository and main branch
run: |
set -euo pipefail
test "$GITHUB_REPOSITORY" = "mbeacom/adrkit"
test "$GITHUB_REF" = "refs/heads/main"

recover:
needs: validate-context
if: >-
github.repository == 'mbeacom/adrkit' &&
github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
environment: npm
permissions:
actions: read
contents: write
steps:
- name: Resolve the requested stable release
id: release
env:
RELEASE_TAG: ${{ inputs.tag }}
run: |
set -euo pipefail
if [[ ! "$RELEASE_TAG" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then
echo "Expected a stable lockstep tag v<semver>, got '$RELEASE_TAG'." >&2
exit 1
fi
version="${RELEASE_TAG#v}"
{
echo "tag=$RELEASE_TAG"
echo "version=$version"
} >> "$GITHUB_OUTPUT"

- name: Check out trusted recovery tooling
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false

- name: Set up Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version: 1.3.14

- name: Require an existing successful stable release
id: source
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ steps.release.outputs.tag }}
RELEASE_VERSION: ${{ steps.release.outputs.version }}
run: |
set -euo pipefail
test "$(git cat-file -t "refs/tags/$RELEASE_TAG")" = tag
revision=$(git rev-parse "$RELEASE_TAG^{commit}")
test "$(git show "$revision:package.json" | jq -r .version)" = "$RELEASE_VERSION"
git cat-file -e "$revision:packages/ci/action.yml"
git cat-file -e "$revision:packages/ci/dist/index.js"
git cat-file -e "$revision:packages/ci/queue/action.yml"
git cat-file -e "$revision:packages/ci/dist/queue-action.js"

git fetch --no-tags origin main
git merge-base --is-ancestor "$revision" origin/main

release=$(gh release view "$RELEASE_TAG" \
--repo "$GITHUB_REPOSITORY" \
--json isDraft,isPrerelease)
test "$(jq -r .isDraft <<<"$release")" = false
test "$(jq -r .isPrerelease <<<"$release")" = false

runs=$(gh api \
"repos/$GITHUB_REPOSITORY/actions/workflows/release.yml/runs?event=push&head_sha=$revision&per_page=100")
successful_run=$(jq -r --arg revision "$revision" --arg tag "$RELEASE_TAG" \
'[.workflow_runs[] | select(.head_sha == $revision and .head_branch == $tag and .conclusion == "success")][0].id // empty' \
<<<"$runs")
test -n "$successful_run"
echo "revision=$revision" >> "$GITHUB_OUTPUT"

moving_refs=$(git ls-remote --tags origin refs/tags/v0 refs/tags/v0^{} || true)
moving_ref_sha=$(awk '$2 == "refs/tags/v0" { print $1 }' <<<"$moving_refs")
moving_commit_sha=$(awk '$2 == "refs/tags/v0^{}" { print $1 }' <<<"$moving_refs")
moving_commit_sha=${moving_commit_sha:-$moving_ref_sha}
echo "moving_ref_sha=$moving_ref_sha" >> "$GITHUB_OUTPUT"
echo "moving_commit_sha=$moving_commit_sha" >> "$GITHUB_OUTPUT"

- name: Restore the moving major Action tag
env:
GH_TOKEN: ${{ github.token }}
RELEASE_TAG: ${{ steps.release.outputs.tag }}
run: |
# The write credential exists in Git config only for this step and is
# removed even when the lease-protected push fails.
set -euo pipefail
header="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n')"
cleanup() {
git config --local --unset-all 'http.https://github.com/.extraheader' || true
}
trap cleanup EXIT
git config --local 'http.https://github.com/.extraheader' "$header"
if [ -n "${{ steps.source.outputs.moving_commit_sha }}" ] &&
[ "${{ steps.source.outputs.moving_commit_sha }}" != "${{ steps.source.outputs.revision }}" ]; then
marker="action-recovery-block/${{ steps.source.outputs.moving_commit_sha }}"
if ! git show-ref --verify --quiet "refs/tags/$marker"; then
git -c tag.gpgSign=false tag "$marker" "${{ steps.source.outputs.moving_commit_sha }}"
fi
git push origin "refs/tags/$marker:refs/tags/$marker"
fi
bun run release:action-tag -- --recover "$RELEASE_TAG" \
--expected-remote-ref-sha "${{ steps.source.outputs.moving_ref_sha }}" |
tee action-tag-recovery.log
{
echo "### Major Action tag recovery"
echo
echo "- Requested release: \`$RELEASE_TAG\`"
echo "- Resolved commit: \`${{ steps.source.outputs.revision }}\`"
echo
echo '```text'
cat action-tag-recovery.log
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
35 changes: 34 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,34 @@ jobs:

- name: Require the release commit on main
run: |
set -euo pipefail
git fetch --no-tags origin main
git merge-base --is-ancestor "$GITHUB_SHA" origin/main
if [ "${{ steps.scope.outputs.lockstep }}" = "true" ]; then
test "$(git cat-file -t "refs/tags/$GITHUB_REF_NAME")" = tag
fi

- name: Refuse a withdrawn lockstep release
if: steps.scope.outputs.lockstep == 'true'
run: |
set -euo pipefail
marker="refs/tags/action-recovery-block/$GITHUB_SHA"
if git ls-remote --exit-code --refs origin "$marker" >/dev/null; then
marker_status=0
else
marker_status=$?
fi
case "$marker_status" in
0)
echo "Lockstep release commit $GITHUB_SHA was withdrawn from the moving Action tag." >&2
exit 1
;;
2) ;;
*)
echo "Unable to check withdrawal marker $marker (git ls-remote exit $marker_status)." >&2
exit "$marker_status"
;;
esac

- name: Install dependencies
run: bun install --frozen-lockfile
Expand Down Expand Up @@ -177,4 +203,11 @@ jobs:
}
trap cleanup EXIT
git config --local 'http.https://github.com/.extraheader' "$header"
bun run release:action-tag -- "$GITHUB_REF_NAME"
bun run release:action-tag -- "$GITHUB_REF_NAME" | tee action-tag-update.log
{
echo "### Moving major Action tag"
echo
echo '```text'
cat action-tag-update.log
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
20 changes: 20 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,26 @@ are reported as `corpus.file-skipped` corpus findings at **`warn`** severity, so
`proposed` record never disappears from the queue silently. Being `warn`, they do
not change the exit code and do not fail the managed-issue Action.

## Moving Action tag recovery

Normal lockstep releases move the lightweight major Action tag (`v0`) forward
only after npm publication and GitHub release creation succeed.
`.github/workflows/action-tag-recovery.yml` is the explicit backward path. Run it
from `main` with an existing stable `vX.Y.Z` release tag. It requires an annotated
tag that peels to a commit on `main`, an exact successful `Release` run, matching
root version, and both committed Action bundles. It shares the release concurrency
group, holds only `actions: read` and `contents: write`, and pushes with a lease
against the observed remote tag object.
Recovery also records a durable `action-recovery-block/<commit>` tag for the
commit removed from `v0`; the normal release workflow rejects a rerun of that
commit before npm publication. A context-validation job fails dispatches from
another repository or ref instead of leaving a skipped workflow green.

Moving `v0` stops future jobs from resolving a bad release; it does not undo an
already-edited PR comment or change a job that already resolved the old SHA.
Restore comment content from GitHub's edit history or rerun the known-good Action.
The full preferred and manual fallback runbook is in `docs/RELEASING.md`.

## The agent plugin (`packages/adapters/agent-plugin`)

The `adrkit` plugin is the fourth distribution surface and the one that reaches
Expand Down
8 changes: 8 additions & 0 deletions docs/DISTRIBUTION.md
Original file line number Diff line number Diff line change
Expand Up @@ -615,6 +615,14 @@ current target (see `docs/RELEASING.md`, "Subsequent releases").
contains `packages/ci/queue/action.yml`, and adopters can switch the pin from the
commit SHA to `@v0`.

Forward promotion is monotonic. If a verified release must be restored instead,
dispatch `.github/workflows/action-tag-recovery.yml` from `main`; it validates
the stable GitHub release and successful release run, peels the annotated version
tag to its commit, shares promotion concurrency with `release.yml`, and moves the
lightweight major tag with a remote-SHA lease. The full containment and comment
restoration runbook is in `docs/RELEASING.md`, "Recovering the moving major
Action tag."

**Done.** v0.2.1 executed exactly this: `v0` and `v0.2.1^{}` both peel to
`31bed03a179b6bfa4a62f7e69008c7441c62598f`, and
`GET /repos/mbeacom/adrkit/contents/packages/ci/queue/action.yml?ref=v0` returns the
Expand Down
115 changes: 115 additions & 0 deletions docs/RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -385,6 +385,121 @@ merge and the tag the site is deployed claiming a release that does not exist ye
The window is short and self-correcting; it is called out here so it is not
mistaken for a mistake.

## Recovering the moving major Action tag

The repository-backed Actions are consumed through a moving lightweight tag,
currently `mbeacom/adrkit/packages/ci@v0` and
`mbeacom/adrkit/packages/ci/queue@v0`. A bad move has a broad but bounded blast
radius: new jobs using `@v0` can resolve the bad release, but the Action cannot
delete repository content or approve a change.

Moving `v0` back **stops future jobs from resolving the bad release; it does not
undo work a completed job already performed**. In particular, it does not restore
an already-edited PR comment. Restore that content from GitHub's comment edit
history, or rerun the known-good Action after recovery so it replaces its managed
comment. Cancel still-running release or consumer jobs when immediate containment
matters: a job that already resolved or checked out the old SHA can continue using
it even after the tag moves.

### Preferred guarded recovery

Choose the last verified lockstep release, then dispatch the recovery workflow
from `main`:

```sh
target=v0.10.0
gh workflow run action-tag-recovery.yml --ref main -f tag="$target"
gh run list --workflow action-tag-recovery.yml --limit 1
```

The workflow refuses a prerelease, draft, lightweight tag, tag whose commit is
not on `main`, release without a successful `Release` run for the exact peeled
commit, root-version mismatch, or tree without both committed Action bundles.
Stable release tags are annotated objects, so the workflow resolves the commit
with `git rev-parse "$target^{commit}"`; the annotated tag object's own SHA is
not a runnable Action revision.

Recovery uses only `actions: read` and `contents: write`, checks out with
credentials disabled, and exposes the write credential only during the final
push. It shares `release-${{ github.repository }}` concurrency with the normal
release workflow, so rollback and forward promotion cannot overlap. The push is
guarded by `--force-with-lease` against the exact remote tag object observed
during validation. If another actor moves the tag despite serialization, recovery
fails rather than overwriting that change.

Both forward promotion and recovery record the prior release tag/commit and the
new release tag/commit in the run summary. Verify the result independently:

```sh
target_commit=$(git rev-parse "$target^{commit}")
test "$(git ls-remote --tags origin refs/tags/v0 | awk '{print $1}')" = "$target_commit"
gh api repos/mbeacom/adrkit/git/ref/tags/v0 \
--jq '{type: .object.type, sha: .object.sha}'
```

The API should report a lightweight tag (`type: commit`) at `target_commit`.
Consumers do not need to change their workflow files. New runs resolve the moved
tag; rerun any job that had already resolved the bad SHA. Consumers needing an
immediate immutable containment pin can temporarily use `@<target_commit>`.

### Manual fallback when GitHub Actions is unavailable

Use a clean checkout of current `main` and a credential limited to this
repository with **Actions: read** and **Contents: write**. Do not hand-write a
plain `git tag -f` / `git push --force` sequence: it omits the release guards and
can overwrite a concurrent promotion.

```sh
set -euo pipefail
target=v0.10.0
git fetch --no-tags origin main "refs/tags/$target:refs/tags/$target"
test "$(git cat-file -t "refs/tags/$target")" = tag
target_commit=$(git rev-parse "$target^{commit}")
test "$(git show "$target_commit:package.json" | jq -r .version)" = "${target#v}"
git cat-file -e "$target_commit:packages/ci/action.yml"
git cat-file -e "$target_commit:packages/ci/dist/index.js"
git cat-file -e "$target_commit:packages/ci/queue/action.yml"
git cat-file -e "$target_commit:packages/ci/dist/queue-action.js"
git merge-base --is-ancestor "$target_commit" origin/main

release=$(gh release view "$target" --json isDraft,isPrerelease)
test "$(jq -r .isDraft <<<"$release")" = false
test "$(jq -r .isPrerelease <<<"$release")" = false
runs=$(gh api \
"repos/mbeacom/adrkit/actions/workflows/release.yml/runs?event=push&head_sha=$target_commit&per_page=100")
test "$(jq -r --arg sha "$target_commit" --arg tag "$target" \
'[.workflow_runs[] | select(.head_sha == $sha and .head_branch == $tag and .conclusion == "success")] | length' \
<<<"$runs")" -gt 0

moving_refs=$(git ls-remote --tags origin refs/tags/v0 refs/tags/v0^{} || true)
moving_ref_sha=$(awk '$2 == "refs/tags/v0" { print $1 }' <<<"$moving_refs")
moving_commit_sha=$(awk '$2 == "refs/tags/v0^{}" { print $1 }' <<<"$moving_refs")
moving_commit_sha=${moving_commit_sha:-$moving_ref_sha}
if [ -n "$moving_commit_sha" ] && [ "$moving_commit_sha" != "$target_commit" ]; then
marker="action-recovery-block/$moving_commit_sha"
if ! git show-ref --verify --quiet "refs/tags/$marker"; then
git -c tag.gpgSign=false tag "$marker" "$moving_commit_sha"
fi
git push origin "refs/tags/$marker:refs/tags/$marker"
fi

bun run release:action-tag -- --recover "$target" \
--expected-remote-ref-sha "$moving_ref_sha"
```

This fallback performs the same package-version, four-bundle, annotated-tag,
main-ancestry, stable-release, and exact successful-run checks as the workflow.
It also records a durable withdrawal marker for the commit being removed from
`v0`; the normal release workflow refuses any later rerun of that withdrawn
commit before npm publication. The script allows recovery from an arbitrary
current `v0` target, but normal `release:action-tag` calls remain monotonic and
cannot bypass the marker gate.

This recovery is intentionally separate from npm rollback. npm versions and
immutable `vX.Y.Z` git tags never move; deprecate a bad npm version, optionally
move npm's `latest` dist-tag for containment, and publish a higher hotfix as
described in [Recovering a bad npm release](#recovering-a-bad-npm-release).

## OCI container image

[ADR-0032](adr/0032-publish-one-lockstep-oci-image-after-the-coordinated-release-succeeds.md)
Expand Down
Loading
Loading