From 6af8d975875b913e551b16263d663e68b10b3d53 Mon Sep 17 00:00:00 2001 From: Yotam Fromm Date: Mon, 7 Sep 2026 08:11:08 +0300 Subject: [PATCH 1/6] fix(ci): recover stories Shortcut knows about and reconcile the Deploy Ready backlog The prod release-notes pipeline resolved shipped stories in one direction only -- git to Shortcut -- and only ever within the current prod tag range. Two consequences, both observed live against the 20 stories standing in Deploy Ready after the 7.1.3 rollout: - A PR whose branch and commit subject carry no story code was invisible, even though Shortcut itself held the PR-to-story link. (1 of 20.) - Anything that shipped in an earlier range was never revisited and stayed in Deploy Ready permanently. (10 of 20 -- the dominant cause.) shipped_stories.py gains Shortcut's PR link as a third discovery source, resolved via `search/stories?query=pr:`, feeding both the transition and the announcement prose. reconcile_deploy_ready.py (new) sweeps every non-archived Deploy Ready story org-wide each release and transitions those whose linked PR already reached prod, whichever release shipped it. Enumeration uses the token'd search endpoint, which -- unlike iterations-get-active -- is not scoped to the caller's own teams. Shipping evidence is a PR merge commit being an ancestor of the prod tag, not a commit-subject grep. It is dry-run by default and never writes into shipped-stories.json: those stories shipped in earlier releases, and leaking them into today's announcement would have Slack claim old features shipped today. Its report goes to $RUNNER_TEMP rather than the checkout, because the prose agent holds Glob+Read over its working directory and a distinct filename is a naming convention, not an access boundary. Three guards decide whether a linked PR counts as evidence -- merged, the right repository, and targeting master. Each caught a real false positive: a story linked to a PR in another repo resolves against this one as an unrelated older PR, and a promotion merge resolves to a story just as readily as that story's real feature PR does. They live in one shared module because the first implementation applied them to the sweep but not to the new fallback, and the two drifted immediately. Stories with no resolvable PR are reported for human triage rather than guessed at. Tests are not collected by pytest.ini and must be run by path with `-p no:django -c /dev/null`; see the README. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KJqBksQqzHYs3F54Y4tJRX --- .github/workflows/prod-release-notes.yaml | 78 ++- build/ci/README-prod-release-notes.md | 224 +++++++- build/ci/reconcile_deploy_ready.py | 540 ++++++++++++++++++ build/ci/shipped_stories.py | 225 +++++++- build/ci/shortcut_pr_guards.py | 88 +++ build/ci/tests/test_reconcile_deploy_ready.py | 524 +++++++++++++++++ build/ci/tests/test_shipped_stories.py | 373 ++++++++++++ 7 files changed, 2020 insertions(+), 32 deletions(-) create mode 100644 build/ci/reconcile_deploy_ready.py create mode 100644 build/ci/shortcut_pr_guards.py create mode 100644 build/ci/tests/test_reconcile_deploy_ready.py diff --git a/.github/workflows/prod-release-notes.yaml b/.github/workflows/prod-release-notes.yaml index d91234804e..abf962e24d 100644 --- a/.github/workflows/prod-release-notes.yaml +++ b/.github/workflows/prod-release-notes.yaml @@ -15,11 +15,22 @@ name: "Prod Release Notes" # 2. build/ci/mark_stories_deployed.py moves each of those stories from # Deploy Ready -> Done via the Shortcut API. A failure here (missing # token, API error, nothing to move) is surfaced as a warning but never -# blocks steps 3-4 — see "Mark shipped stories as deployed" below. -# 3. The sefaria-release-notes skill (.claude/skills/sefaria-release-notes/) +# blocks steps 3-5 — see "Mark shipped stories as deployed" below. +# 3. build/ci/reconcile_deploy_ready.py sweeps EVERY non-archived Deploy +# Ready story org-wide (not just this release's commit range) and +# transitions any whose linked PR already reached prod, regardless of +# which release actually shipped it. This backfills stories a prior +# release's git-range-scoped run never revisited (RC2) — see that +# script's own docstring for the four guards it applies. Its output +# NEVER feeds shipped-stories.json or step 4/5 below: those stories +# shipped in EARLIER releases, and leaking them into today's +# announcement would have Slack claim old features shipped today. A +# failure here is warned/alerted the same way step 2's is, and never +# blocks steps 4-5. +# 4. The sefaria-release-notes skill (.claude/skills/sefaria-release-notes/) # reads that JSON and writes prose only — it does not talk to GitHub or # Shortcut. -# 4. scripts/post_to_slack.py posts both generated files to Slack. +# 5. scripts/post_to_slack.py posts both generated files to Slack. # # Manual setup this depends on: see build/ci/README-prod-release-notes.md @@ -149,19 +160,72 @@ jobs: fi python3 build/ci/mark_stories_deployed.py "${ARGS[@]}" + - name: Reconcile Deploy Ready backlog + id: reconcile + # Separate concern from this release's own commit-range bookkeeping + # above: this sweeps EVERY non-archived Deploy Ready story org-wide + # and transitions any whose linked PR already reached prod, + # regardless of which release shipped it (RC2 — see + # reconcile_deploy_ready.py's own docstring for the four guards it + # applies before trusting a linked PR). Its report file is never + # read by the release-notes/Slack steps below — see the workflow + # header comment for why that separation is load-bearing. It's + # written to $RUNNER_TEMP rather than the checkout (GITHUB_WORKSPACE) + # specifically so it can't leak into the prose step: that step's + # prompt only NAMES shipped-stories.json, but the agent still holds + # Glob+Read over its whole working directory, and a same-directory + # JSON full of real story names is exactly the "old stories leak + # into today's announcement" failure this whole separation exists to + # prevent — a distinct filename is a naming convention, not an + # access boundary. A failure here (Shortcut API hiccup, gh + # unreachable, a transition error) must never skip release-notes + # generation and Slack posting for an otherwise healthy rollout, so + # this gets the same continue-on-error treatment as "Mark shipped + # stories as deployed" above, and the two outcomes are + # warned/alerted together by the two steps that follow. + continue-on-error: true + env: + SHORTCUT_API_TOKEN: ${{ secrets.SHORTCUT_API_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EVENT_NAME: ${{ github.event_name }} + DRY_RUN_INPUT: ${{ inputs.dry_run }} + run: | + ARGS=(--out "$RUNNER_TEMP/reconcile-deploy-ready-report.json") + if [[ "$EVENT_NAME" != "workflow_dispatch" || "$DRY_RUN_INPUT" != "true" ]]; then + ARGS+=(--apply) + fi + python3 build/ci/reconcile_deploy_ready.py "${ARGS[@]}" + - name: Warn if Shortcut bookkeeping failed - if: steps.mark_deployed.outcome == 'failure' + if: steps.mark_deployed.outcome == 'failure' || steps.reconcile.outcome == 'failure' + env: + MARK_DEPLOYED_OUTCOME: ${{ steps.mark_deployed.outcome }} + RECONCILE_OUTCOME: ${{ steps.reconcile.outcome }} run: | - echo "::warning::mark_stories_deployed.py failed -- shipped Shortcut stories were NOT moved Deploy Ready -> Done for this release. Release notes generation is continuing anyway. Check this run's 'Mark shipped stories as deployed' step and move the affected stories manually." + if [[ "$MARK_DEPLOYED_OUTCOME" == "failure" ]]; then + echo "::warning::mark_stories_deployed.py failed -- shipped Shortcut stories were NOT moved Deploy Ready -> Done for this release. Release notes generation is continuing anyway. Check this run's 'Mark shipped stories as deployed' step and move the affected stories manually." + fi + if [[ "$RECONCILE_OUTCOME" == "failure" ]]; then + echo "::warning::reconcile_deploy_ready.py failed -- the org-wide Deploy Ready backlog sweep did not complete. Release notes generation is continuing anyway. Check this run's 'Reconcile Deploy Ready backlog' step; the backlog will be retried on the next release." + fi - name: Alert Slack if Shortcut bookkeeping failed - if: steps.mark_deployed.outcome == 'failure' + if: steps.mark_deployed.outcome == 'failure' || steps.reconcile.outcome == 'failure' env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOY_WEBHOOK }} VERSION: ${{ steps.resolve.outputs.version }} + MARK_DEPLOYED_OUTCOME: ${{ steps.mark_deployed.outcome }} + RECONCILE_OUTCOME: ${{ steps.reconcile.outcome }} run: | + FAILURES="" + if [[ "$MARK_DEPLOYED_OUTCOME" == "failure" ]]; then + FAILURES="${FAILURES}mark_stories_deployed.py failed (this release's shipped stories NOT moved Deploy Ready -> Done). " + fi + if [[ "$RECONCILE_OUTCOME" == "failure" ]]; then + FAILURES="${FAILURES}reconcile_deploy_ready.py failed (org-wide Deploy Ready backlog sweep did not complete). " + fi curl -s -X POST -H 'Content-type: application/json' \ - --data "{\"text\": \":warning: Prod release notes for version ${VERSION:-unknown}: mark_stories_deployed.py failed -- shipped Shortcut stories were NOT moved Deploy Ready -> Done. Release notes generation is continuing. Check the workflow run and move the affected stories manually.\"}" \ + --data "{\"text\": \":warning: Prod release notes for version ${VERSION:-unknown}: ${FAILURES}Release notes generation is continuing. Check the workflow run and move the affected stories manually.\"}" \ "$SLACK_WEBHOOK_URL" || true - name: Set up Node diff --git a/build/ci/README-prod-release-notes.md b/build/ci/README-prod-release-notes.md index ebdee55e7c..34ad9c36f0 100644 --- a/build/ci/README-prod-release-notes.md +++ b/build/ci/README-prod-release-notes.md @@ -11,25 +11,189 @@ Argo post-promotion analysis (prod) -> repository_dispatch (prod-rollout-succeeded, carries `version` + `chartVersion`) -> build/ci/shipped_stories.py — walks the prod/* tag range in git, resolves Shortcut story codes from - commit subjects / merged-PR branch - names, hydrates story details + commit subjects, merged-PR branch + names, AND (as a third, fallback + source) Shortcut's own PR<->story + link, hydrates story details -> build/ci/mark_stories_deployed.py — moves each shipped story Deploy Ready -> Done via the Shortcut API. A failure here (missing token, API error, nothing to move) is logged and Slack-alerted but never - blocks the two steps below. - -> sefaria-release-notes skill — reads the JSON, writes prose only + blocks the steps below. + -> build/ci/reconcile_deploy_ready.py — separately, sweeps EVERY + non-archived Deploy Ready story + org-wide (not just this release's + commit range) and transitions any + whose linked PR already reached prod + — see "Reconciliation sweep" below. + Its output NEVER reaches the two + steps that follow. + -> sefaria-release-notes skill — reads shipped-stories.json, writes + prose only -> scripts/post_to_slack.py — posts both files to Slack ``` Only the release-notes generation step is an LLM. Deciding which stories a given deploy closed -is a graph walk over git history and Shortcut IDs, and flipping their -workflow state is a for-loop over a REST API — neither of those is a job -for a model. The skill's only input is the JSON that +is a graph walk over git history, Shortcut IDs and the Shortcut API, and +flipping a story's workflow state is a for-loop over a REST API — none of +that is a job for a model. The skill's only input is the JSON that `shipped_stories.py` already produced; it does not call GitHub or Shortcut itself, and it does not mutate any story. +## Three discovery sources in `shipped_stories.py` + +For each commit in the resolved tag range, a story id is looked for in, in +order: + +1. **The commit subject itself** (`sc-NNNNN` in any of its usual shapes — + `fix(sc-123):`, `[sc-123]`, `feature/sc-123`, ...). +2. **The branch name of the commit's merged PR**, when the commit carries a + `(#N)` reference or is a bare "Merge pull request #N from ..." — some + teams put the story code in the branch instead of the commit message. +3. **Shortcut's own PR<->story link** (`GET search/stories?query=pr:`), + used ONLY as a fallback for a commit whose PR carries no story id from + either source above. This exists because git text is not the only place + a story/PR link can live — a story can be attached to a PR from the + Shortcut UI with no story code ever appearing in the branch name. + `branch:"..."` and `pull-request:N` do NOT resolve this on this org; + only the `pr:N` search operator does. The id is adopted ONLY when the + search returns EXACTLY one story — an ambiguous match (>1) is logged and + skipped rather than guessed. + + **This fallback is guarded the same way `reconcile_deploy_ready.py`'s + sweep is (via the shared `shortcut_pr_guards.py`), and for the same + reason: a bare `pr:` match only proves Shortcut linked SOME story to + that PR number, not that the PR is real shipping evidence.** Verified + live: a promotion PR (head branch `master`/`preprod`/`prod`, merging + into the next environment) resolves via `pr:` to a real story just as + readily as that story's actual feature PR does, while proving nothing + about whether that story's own change shipped. So the fallback (a) is + never even attempted for a commit whose subject is auto-generated + merge/branch-sync noise (`NOISE_PATTERN` — the same pattern already used + to keep such commits out of `commits_without_story`) or whose PR's own + head branch is a long-lived environment branch, and (b) re-checks the + single search result's own linked-PR entry against the three PR-level + guards (merged / Sefaria-Project repo / target branch master) before + adopting it — a match that fails those guards is a warn-and-skip. + + Ids recovered this way are echoed separately in the output's + `stories_from_shortcut_pr_link` list (in + addition to the ordinary `story_ids`) so a report can call out what only + Shortcut knew. Gated on `SHORTCUT_API_TOKEN`; without it (or on any + per-lookup failure) this step is skipped/warned and the run continues + with git-only discovery — it never aborts `shipped_stories.py`. + +## Reconciliation sweep: `build/ci/reconcile_deploy_ready.py` + +`shipped_stories.py` + `mark_stories_deployed.py` only ever look at ONE +release's commit range (`prev-tag..cur-tag`). A story whose PR merged and +shipped in an EARLIER release — or before this pipeline existed — never +gets revisited by that pair of scripts; nothing ever walks backward and +re-checks a story sitting in Deploy Ready. Of the stories stuck in Deploy +Ready when this was diagnosed, ten times as many were this class of gap as +were the discovery gap `shipped_stories.py`'s third source fixes above. + +`reconcile_deploy_ready.py` is a standalone, org-wide sweep that closes +that gap. It enumerates every non-archived Deploy Ready story, resolves +each one's linked merged PR(s), and checks whether any of those PRs' +merge commits are an ancestor of the current prod tag. It classifies every +story into exactly one of three buckets: + +- **shipped** — at least one qualifying PR is in prod. Transitioned + Deploy Ready (500000045) -> Done (500000010). +- **pending** — has a qualifying merged PR, but none are in prod yet. Left + alone — this is the correct state, not a bug. +- **triage** — no qualifying PR at all, or the story lives in a + non-Standard Shortcut workflow. Left alone and reported; this is the + part of the output a human actually has to look at. + +### The four guards + +Each of these caught a real false positive while this script was verified +against live data — skipping any one of them silently mis-transitions a +story. The first three (merged / repo / target branch) are PR-level checks +shared with `shipped_stories.py`'s own RC1 PR-link fallback via +`build/ci/shortcut_pr_guards.py` — both scripts ask the same underlying +question ("does this linked PR actually prove a story's change reached +prod?") and a promotion PR is exactly as good at fooling either one, so +there is exactly one implementation of these three checks, not two +parallel copies that could silently drift apart: + +1. **`repository_id` must be Sefaria-Project's (`500000103`).** A story can + link a PR from a different repo; resolving that PR number against + Sefaria-Project instead finds an unrelated (often much older) PR that + happens to share the number — and that PR can easily already be in + prod, which would report "shipped" for a story that never touched this + repo. +2. **`target_branch_name` must be `"master"`.** Some stories link a + promotion PR (preprod -> prod, or master -> preprod) instead of, or + alongside, the actual feature PR. A promotion PR merges constantly and + proves nothing about whether this story's own change reached prod. +3. **`merged` must be `true`.** An open or closed-without-merging PR is not + evidence anything shipped. +4. **`workflow_id` must be the Standard workflow (`500000005`), and + `workflow_state_id` must be exactly the numeric Deploy Ready id + (`500000045`).** The Shortcut state named "Deploy Ready" — note its real + name carries a trailing space, `"Deploy Ready "` — is workflow-specific: + `500000045` doesn't exist as a concept in, say, the Content workflow. + Enumeration is keyed on the state NAME (the search endpoint has no other + way to filter it), so classification re-checks the NUMERIC ids before + trusting a match; a story on any other workflow, or at any other state + id despite matching the name, is routed to triage with its actual + workflow/state ids reported — mirroring `mark_stories_deployed.py`'s + `skipped_different_workflow` handling. + +Enumeration uses the token'd search endpoint +(`search/stories?query=state:"Deploy Ready" !is:archived`), paginated via +its `next` cursor. This is deliberately NOT `iterations-get-active` — that +endpoint is silently scoped to the calling token's own teams and has +already produced an incomplete picture for this team once; the search +endpoint returns every matching story across every team. + +A qualifying PR's merge commit is resolved via `gh pr view --json +mergeCommit` and tested with `git merge-base --is-ancestor ` — verified to correctly discriminate a merged-but-not-yet-promoted PR +from one that already reached prod. `git log --grep="(#N)"` was tried and +rejected: it misses squash-merge subjects and can't tell a real promotion +merge apart from an unrelated one. + +### Dry-run by default + +Unlike `mark_stories_deployed.py` (which mutates by default and needs +`--dry-run` to preview), `reconcile_deploy_ready.py` inverts that: **it +never mutates anything unless you pass `--apply`.** This is a bulk mutation +of shared state across potentially many stories and several different +teams, and — unlike a single release's handful of stories — there's no +natural moment (a deploy just happened) that makes running it low-risk. An +explicit `--dry-run` flag also exists, purely for symmetry with +`mark_stories_deployed.py` and CI readability; it's a no-op since dry-run +is already the default, and it always wins if both flags are passed +together. + +``` +python3 build/ci/reconcile_deploy_ready.py --dry-run # classify + report, mutate nothing (default) +python3 build/ci/reconcile_deploy_ready.py --apply # actually transition the "shipped" bucket +python3 build/ci/reconcile_deploy_ready.py --apply --prod-tag prod/6.111.0-prod.2+chart.0.87.5-prod.1 --out report.json +``` + +**Critical: this script never reads or writes `shipped-stories.json` and +never feeds the release-notes prose step.** The stories it backfills +shipped in EARLIER releases — leaking them into today's release +announcement would have Slack claim a dozen old features shipped today. +Reconciliation transitions Shortcut state only; it has no opinion about +what today's release notes should say. In the workflow, its step runs +after `mark_stories_deployed.py` and writes its own separate report file +to `$RUNNER_TEMP` (NOT the checkout / `GITHUB_WORKSPACE`) — a distinct +filename alone is a naming convention, not an access boundary, and the +release-notes step's headless Claude run holds `Glob`+`Read` over its whole +working directory, so a same-directory JSON full of real, recently-shipped +story names would be one bad glob away from leaking into the prose it +writes. Keeping the report outside the checkout entirely is what actually +enforces the separation. A failure here is warned/Slack-alerted the same way a +`mark_stories_deployed.py` failure is, and never blocks release-notes +generation or posting. + ## What's already wired up in this repo - `helm-chart/sefaria/templates/analysistemplate/rollout-complete.yaml` — @@ -45,9 +209,14 @@ itself, and it does not mutate any story. - `.github/workflows/prod-release-notes.yaml` — listens for that dispatch (or a manual `workflow_dispatch`), resolves the version (and optional chart version, for disambiguating a chart-only rollout), runs - `shipped_stories.py` and `mark_stories_deployed.py`, runs the - `sefaria-release-notes` skill headlessly, and posts both output files to - Slack via `scripts/post_to_slack.py`. + `shipped_stories.py` and `mark_stories_deployed.py`, separately runs + `reconcile_deploy_ready.py` (its report never reaches the steps below), + runs the `sefaria-release-notes` skill headlessly, and posts both output + files to Slack via `scripts/post_to_slack.py`. The reconcile step honors + the same `workflow_dispatch` `dry_run` input as `mark_stories_deployed.py` + does — real trigger or `dry_run=false` passes `--apply`; the default + `workflow_dispatch` (`dry_run=true`) leaves it in its default dry-run + mode. - `.claude/skills/sefaria-release-notes/` — the skill, shipped in-repo, now takes a shipped-stories JSON file as its only input and only writes prose. It no longer talks to GitHub or Shortcut. @@ -83,7 +252,7 @@ Add these under repo Settings → Secrets and variables → Actions: | Secret | Purpose | Notes | |---|---|---| -| `SHORTCUT_API_TOKEN` | `shipped_stories.py` story hydration and `mark_stories_deployed.py` state transitions | Shortcut → Settings → API Tokens. Not the same as the OAuth MCP connection used interactively. | +| `SHORTCUT_API_TOKEN` | `shipped_stories.py` story hydration and PR-link fallback, `mark_stories_deployed.py` state transitions, `reconcile_deploy_ready.py` enumeration and transitions | Shortcut → Settings → API Tokens. Not the same as the OAuth MCP connection used interactively. | | `SLACK_PRODUCT_WEBHOOK` | Non-technical release announcement | A second Slack incoming webhook, pointed at whichever channel should get `release-announcement-product-slack.txt`. Until this is set, that post step is a guarded no-op (won't fail the workflow). | Already exist and are reused as-is: `SLACK_DEPLOY_WEBHOOK`, `GITHUB_TOKEN`, @@ -133,3 +302,36 @@ anywhere — it would just be quiet. 5. Confirm both Slack files post correctly, and confirm the shipped stories actually moved Deploy Ready → Done in Shortcut. + +## Running the tests + +The tests for these scripts (`build/ci/tests/test_shipped_stories.py`, +`test_mark_stories_deployed.py`, `test_reconcile_deploy_ready.py`) are +**not** collected by the repo's root `pytest.ini` (that config is scoped to +the Django app's own test suites), so run them by explicit path from the +repo root: + +``` +python3 -m pytest build/ci/tests/test_shipped_stories.py build/ci/tests/test_mark_stories_deployed.py build/ci/tests/test_reconcile_deploy_ready.py -q -p no:django -c /dev/null +``` + +Both flags are needed even though only explicit file paths are passed: +`pytest` still discovers and loads the repo-root `pytest.ini` from the +current directory regardless of which paths are given on the command line, +and that ini sets `DJANGO_SETTINGS_MODULE` — which makes the `pytest-django` +plugin try to `django.setup()` the whole app (and fail with +`ModuleNotFoundError: No module named 'allauth'` in an environment that +hasn't installed the full Django app's dependencies, which these +standalone, stdlib-only scripts have no need of). `-c /dev/null` stops +`pytest.ini` from being read at all; `-p no:django` disables the +`pytest-django` plugin itself as a second, independent line of defense +(matters if some other ini/plugin-autouse path re-enables it). Depending on +which Python environment you invoke `pytest` from, the plain command +without these flags may happen to work (if that environment has the full +Django app's dependencies installed) or may not — the flagged command works +regardless. + +No network access, `git`, or `gh` binary is required — everything that +would otherwise shell out or call the Shortcut API is monkeypatched at the +same boundary the module itself uses (`subprocess.run`, +`urllib.request.urlopen`, or the module's own `run_git`/helper functions). diff --git a/build/ci/reconcile_deploy_ready.py b/build/ci/reconcile_deploy_ready.py new file mode 100644 index 0000000000..a91745c5b0 --- /dev/null +++ b/build/ci/reconcile_deploy_ready.py @@ -0,0 +1,540 @@ +#!/usr/bin/env python3 +""" +Reconcile "Deploy Ready" Shortcut stories against what has actually reached +prod, regardless of which release shipped them (RC2 in the incident +writeup). + +`shipped_stories.py` + `mark_stories_deployed.py` only ever look at the +commit range for ONE release (prev-tag..cur-tag). A story whose PR merged +and shipped in an EARLIER release -- or before this pipeline existed at all +-- never gets revisited: nothing ever walks backward and asks "is this +Deploy Ready story actually done?". Of 20 stories stuck in Deploy Ready when +this was diagnosed, 10 were this class of bug versus 1 for the git-only +discovery gap (RC1, fixed in shipped_stories.py) -- this is the dominant +defect. + +This script is a standalone, org-wide sweep: it enumerates EVERY +non-archived Deploy Ready story (not just stories that happen to fall in +some git range), resolves each one's linked, merged PR(s), and checks +whether any of those PRs' merge commits actually landed in the current prod +tag. It classifies every story into exactly one of three buckets: + + - shipped -- at least one qualifying PR is an ancestor of the prod tag. + Transitioned Deploy Ready -> Done (500000045 -> 500000010). + - pending -- has a qualifying merged PR, but none of them are in prod + yet. Left alone -- this is the CORRECT state, not a bug. + - triage -- no qualifying PR at all (nothing merged, wrong repo, a + promotion PR instead of the real one, ...), or the story + lives in a non-Standard Shortcut workflow where the Deploy + Ready/Done state ids used here don't even apply. Left + alone and reported for a human to look at -- this is the + part of the output that actually needs eyes on it. + +Four guards apply before a linked PR counts as evidence a story shipped -- +each one caught a real false positive while this script was being built +against live data: + + 1. repository_id must be Sefaria-Project's (500000103). A story can link + a PR from a DIFFERENT repo (e.g. a docs or infra repo); resolving that + PR number against Sefaria-Project instead finds an unrelated, often + much older, PR that happens to share the number -- and that PR can + easily already be in prod. Skipping this guard silently reports "in + prod" for a story that never shipped anything to Sefaria-Project at + all. + 2. target_branch_name must be "master". Some stories link a PROMOTION PR + (preprod -> prod, or master -> preprod) instead of, or alongside, the + actual feature PR. A promotion PR merges constantly and proves nothing + about whether THIS story's own change reached prod. + 3. merged must be true. An open or closed-without-merging PR is not + evidence of anything having shipped. + 4. workflow_id must be Sefaria's "Standard" workflow (500000005), and + workflow_state_id must be exactly the numeric Deploy Ready state id + (500000045) within it. The Shortcut state named "Deploy Ready" (note: + the real name carries a trailing space, "Deploy Ready ") is + workflow-specific -- 500000045 does not exist as a concept in, say, + the Content workflow (500000061). A story living in a different + workflow that the search endpoint still matched by state NAME must + never be transitioned using Standard's state ids; it's routed to + triage with its actual workflow/state ids reported instead, mirroring + mark_stories_deployed.py's skipped_different_workflow handling. + +Enumeration uses the token'd search endpoint +(`search/stories?query=state:"Deploy Ready" !is:archived`), paginated via +its `next` cursor -- NOT `iterations-get-active`, which is silently scoped +to the calling token's own teams and has already produced an incomplete +picture for this team once. The search endpoint returned all 20 stuck +stories across 4 different teams in testing; iterations-get-active would +have missed most of them. + +For each qualifying PR, its merge commit SHA is resolved via +`gh pr view --json mergeCommit` and tested with +`git merge-base --is-ancestor ` against the newest `prod/*` +tag (by `--sort=-creatordate`) unless `--prod-tag` overrides it. This is the +same ancestry check verified against live data to correctly discriminate +merged-but-not-yet-promoted PRs (not an ancestor) from ones that already +reached prod (an ancestor) -- `git log --grep="(#N)"` was tried and +rejected: it misses squash-merge subjects and can't tell a real promotion +merge apart from an unrelated one. + +--dry-run is the default (matching mark_stories_deployed.py's posture, only +stricter): this is a bulk mutation of shared team state across potentially +many stories and several different teams, and unlike a single release's +handful of stories, there's no natural moment (a deploy) that makes running +it low-risk. Nothing is ever transitioned without an explicit --apply. + +Emits a JSON report (--out) with all three buckets in full, plus a readable +stdout summary. CRITICAL: this script never reads or writes +shipped-stories.json and never feeds the release-notes prose step -- the +stories it backfills shipped in EARLIER releases, and leaking them into +today's release announcement would have Slack claim a dozen old features +shipped today. Reconciliation transitions Shortcut state only; it has no +opinion about what today's release notes should say. + +Usage: + python3 reconcile_deploy_ready.py [--dry-run] + python3 reconcile_deploy_ready.py --apply + python3 reconcile_deploy_ready.py --apply --prod-tag prod/6.111.0-prod.2+chart.0.87.5-prod.1 --out reconcile-report.json + +Requires `git` and `gh` on PATH (same as shipped_stories.py) and +SHORTCUT_API_TOKEN -- required even for --dry-run, since enumeration and +the PR<->story evidence both come from the Shortcut API, not from git. + +All story ids shown in this file's docstring and comments (e.g. 11111, +22222) are placeholders, not real Shortcut story ids. The repo/workflow/ +state ids (500000103, 500000005, 500000045, 500000010, 500000061) are real +Shortcut/GitHub ids, not story ids, and are not covered by that placeholder +rule -- same convention as shipped_stories.py and mark_stories_deployed.py. +""" + +import argparse +import concurrent.futures +import json +import os +import subprocess +import sys +import urllib.error +import urllib.parse +import urllib.request + +# The PR-level shipping-evidence guards (merged / right repo / right target +# branch) are shared with shipped_stories.py's RC1 PR-link fallback -- see +# shortcut_pr_guards.py's own docstring for why a single shared +# implementation matters here (a promotion PR is exactly as good at fooling +# either script, and the two must never silently drift apart on what counts +# as evidence). build/ci is not a package (see tests/conftest.py), but a +# plain sibling-module import works both when this file is run directly +# (python3 puts its own directory on sys.path[0]) and under pytest (the test +# conftest adds build/ci to sys.path the same way). +import shortcut_pr_guards + +SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" +SHORTCUT_API_ROOT = "https://api.app.shortcut.com" + +# Sefaria's "Standard" Shortcut workflow -- same ids mark_stories_deployed.py +# defaults to. Kept as separate constants here (not imported) because these +# CI scripts are deliberately standalone, single-file tools -- see the +# existing die()/warn() duplication between shipped_stories.py and +# mark_stories_deployed.py, which follows the same convention. (The PR-level +# guard constants/functions are the one deliberate exception -- see the +# shortcut_pr_guards import above.) +STANDARD_WORKFLOW_ID = 500000005 +DEPLOY_READY_STATE_ID = 500000045 +DONE_STATE_ID = 500000010 + +# Guard #1/#2 defaults -- re-exported from shortcut_pr_guards so the rest of +# this file (and its tests) can keep referring to them by their existing +# names here. +SEFARIA_PROJECT_REPO_ID = shortcut_pr_guards.SEFARIA_PROJECT_REPO_ID +DEFAULT_TARGET_BRANCH = shortcut_pr_guards.DEFAULT_TARGET_BRANCH +DEFAULT_REPO = "Sefaria/Sefaria-Project" + +# `branch:"..."` and `pull-request:N` do NOT resolve a PR to its story on +# this Shortcut org -- verified empirically. `pr:N` is the only search +# operator that works, both here and in shipped_stories.fetch_story_by_pr_link. +DEPLOY_READY_SEARCH_QUERY = 'state:"Deploy Ready" !is:archived' + + +def die(message: str) -> None: + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def warn(message: str) -> None: + print(f"WARNING: {message}", file=sys.stderr) + + +def run_git(args): + """Run a git subcommand, returning stdout. Exits the process on failure. + Mirrors shipped_stories.run_git -- used here only for the plain, + always-expected-to-succeed prod-tag listing; the ancestry check below + needs its own non-fatal handling of git's exit codes, so it does not go + through this helper.""" + proc = subprocess.run(["git", *args], capture_output=True, text=True) + if proc.returncode != 0: + die(f"git {' '.join(args)} failed: {proc.stderr.strip()}") + return proc.stdout + + +def resolve_default_prod_tag(): + """Newest prod/* tag by creation date, the same ordering + shipped_stories.py uses for --version resolution. Dies if there are no + prod/* tags at all -- with none, there's no prod state to reconcile + against, and --prod-tag can be passed explicitly if that's ever wrong.""" + tags = [t for t in run_git(["tag", "--list", "prod/*", "--sort=-creatordate"]).splitlines() if t.strip()] + if not tags: + die("No 'prod/*' tags found in this checkout; pass --prod-tag explicitly.") + return tags[0] + + +def search_deploy_ready_stories(token): + """Enumerate ALL non-archived Deploy Ready stories via the token'd + search endpoint, paginated via its `next` cursor. Deliberately NOT + `iterations-get-active`: that endpoint is silently scoped to the + calling token's own teams, while `search/stories` returned all 20 + stuck stories across 4 different teams when this was verified live -- + org-wide is exactly what "every Deploy Ready story" requires.""" + stories = [] + next_path = f"/api/v3/search/stories?query={urllib.parse.quote(DEPLOY_READY_SEARCH_QUERY)}" + while next_path: + url = next_path if next_path.startswith("http") else SHORTCUT_API_ROOT + next_path + req = urllib.request.Request(url, headers={"Shortcut-Token": token, "Accept": "application/json"}) + with urllib.request.urlopen(req, timeout=20) as resp: + body = json.loads(resp.read().decode("utf-8")) + stories.extend(body.get("data", [])) + next_path = body.get("next") + return stories + + +# gather_linked_prs / qualifying_prs (guards #1-3) now live in +# shortcut_pr_guards.py, shared with shipped_stories.py's RC1 fallback -- +# re-exported here under their existing names so nothing else in this file +# (or its tests) needs to change. Guard #4 (workflow/state) is a +# story-level check specific to the Deploy Ready sweep and stays local to +# classify_stories below. +gather_linked_prs = shortcut_pr_guards.gather_linked_prs +qualifying_prs = shortcut_pr_guards.qualifying_prs + + +def fetch_pr_merge_oid(pr_number, repo): + """Resolve a merged PR's merge commit SHA via `gh pr view --json + mergeCommit`. Mirrors shipped_stories.fetch_pr_branch's error posture: a + failed lookup is logged and returns None rather than raising -- one + story's PR being unreachable must never abort the whole sweep.""" + proc = subprocess.run( + ["gh", "pr", "view", str(pr_number), "--repo", repo, "--json", "mergeCommit"], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + warn(f"gh pr view {pr_number} --repo {repo} failed: {proc.stderr.strip()}") + return pr_number, None + try: + data = json.loads(proc.stdout) + except json.JSONDecodeError as e: + warn(f"gh pr view {pr_number} --repo {repo} returned invalid JSON: {e}") + return pr_number, None + oid = (data.get("mergeCommit") or {}).get("oid") + return pr_number, oid + + +def fetch_merge_oids(pr_numbers, repo, max_workers=8): + """Batch-resolve fetch_pr_merge_oid across every qualifying PR in this + sweep concurrently -- same pattern as shipped_stories.fetch_branches.""" + oid_by_pr = {} + if not pr_numbers: + return oid_by_pr + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(fetch_pr_merge_oid, n, repo) for n in pr_numbers] + for future in concurrent.futures.as_completed(futures): + try: + pr_number, oid = future.result() + except FileNotFoundError: + # Every other in-flight lookup will hit the same error -- + # fail fast with one clear message instead of N tracebacks. + die("`gh` was not found on PATH. Install the GitHub CLI " + "(https://cli.github.com/) or ensure it's available in " + "this environment; PR merge-commit lookups cannot proceed without it.") + if oid: + oid_by_pr[pr_number] = oid + return oid_by_pr + + +def is_ancestor_of_prod(oid, prod_tag): + """True if commit `oid` reached prod (is an ancestor of `prod_tag`), + False if git can definitively say it did not. Returns None if the check + itself couldn't run (e.g. a shallow checkout that never fetched `oid`) + -- that's a local-repo problem, not proof either way, and must not be + conflated with a confirmed "not in prod" (which would misclassify a + story that may well have shipped as merely pending).""" + proc = subprocess.run( + ["git", "merge-base", "--is-ancestor", oid, prod_tag], + capture_output=True, + text=True, + ) + if proc.returncode == 0: + return True + if proc.returncode == 1: + return False + warn(f"git merge-base --is-ancestor {oid} {prod_tag} could not run (rc={proc.returncode}): {proc.stderr.strip()}") + return None + + +def transition_story(story_id, done_state_id, token): + """PUT a workflow_state_id update to Shortcut. Mirrors + mark_stories_deployed.transition_story exactly (kept as its own copy + here rather than imported, for the same standalone-script reason the + die()/warn() duplication above follows).""" + url = f"{SHORTCUT_API_BASE}/stories/{story_id}" + body = json.dumps({"workflow_state_id": done_state_id}).encode("utf-8") + req = urllib.request.Request(url, data=body, method="PUT") + req.add_header("Shortcut-Token", token) + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=20) as resp: + resp.read() + return story_id, True, None + except urllib.error.HTTPError as e: + return story_id, False, f"HTTP {e.code} {e.reason}" + except Exception as e: # noqa: BLE001 - a per-story failure must never abort the run + return story_id, False, str(e) + + +def transition_stories(story_ids, done_state_id, token, max_workers=8): + transitioned = [] + failed = [] + if not story_ids: + return transitioned, failed + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(transition_story, sid, done_state_id, token) for sid in story_ids] + for future in concurrent.futures.as_completed(futures): + sid, ok, err = future.result() + if ok: + transitioned.append(sid) + else: + warn(f"Failed to transition story {sid}: {err}") + failed.append((sid, err)) + return transitioned, failed + + +def _story_summary(story): + return {"id": story.get("id"), "name": story.get("name"), "url": story.get("app_url")} + + +def classify_stories(stories, repo_id, target_branch): + """Pure classification, no I/O beyond what's already embedded in the + Shortcut story payloads: split into (triage, candidates), where + candidates is a list of (story, qualifying_prs) pairs still needing a + prod-ancestry check. Guard #4 (workflow + numeric state id) is applied + first and unconditionally routes to triage -- a story on any workflow + other than Standard, or sitting at any state id other than the numeric + Deploy Ready id (500000045) despite matching the "Deploy Ready" state + NAME search, must never reach the PR guards or a transition at all.""" + triage = [] + candidates = [] + for story in stories: + if story.get("workflow_id") != STANDARD_WORKFLOW_ID or story.get("workflow_state_id") != DEPLOY_READY_STATE_ID: + entry = _story_summary(story) + entry["reason"] = "non_standard_workflow_or_state" + entry["workflow_id"] = story.get("workflow_id") + entry["workflow_state_id"] = story.get("workflow_state_id") + triage.append(entry) + warn( + f"Story {story.get('id')} matched the Deploy Ready search but is on " + f"workflow_id={story.get('workflow_id')} / workflow_state_id={story.get('workflow_state_id')} " + f"(expected workflow_id={STANDARD_WORKFLOW_ID}, workflow_state_id={DEPLOY_READY_STATE_ID}); " + "routing to triage, not transitioning." + ) + continue + + linked = gather_linked_prs(story) + qualifying = qualifying_prs(linked, repo_id=repo_id, target_branch=target_branch) + if not qualifying: + entry = _story_summary(story) + entry["reason"] = "no_qualifying_pr" + entry["linked_pr_numbers"] = sorted( + pr.get("number") for pr in linked if pr.get("number") is not None + ) + triage.append(entry) + continue + + candidates.append((story, qualifying)) + return triage, candidates + + +def classify_candidates(candidates, oid_by_pr, prod_tag): + """For each (story, qualifying_prs) candidate, check every qualifying + PR's merge commit for prod ancestry and split into shipped / pending. + A PR whose merge oid never resolved, or whose ancestry check itself + couldn't run, counts as inconclusive -- never as "in prod" -- so a + resolution failure can only ever push a story toward pending (leave it + alone), never wrongly toward shipped (a mutation).""" + shipped = [] + pending = [] + for story, prs in candidates: + shipped_via = [] + for pr in prs: + oid = oid_by_pr.get(pr["number"]) + if not oid: + continue + if is_ancestor_of_prod(oid, prod_tag) is True: + shipped_via.append(pr["number"]) + + entry = _story_summary(story) + entry["qualifying_prs"] = sorted(pr["number"] for pr in prs) + if shipped_via: + entry["shipped_via_prs"] = sorted(shipped_via) + shipped.append(entry) + else: + pending.append(entry) + return shipped, pending + + +def print_summary(report): + """Readable stdout summary. The triage list is printed in FULL, not + just counted -- it's the part of this report a human actually has to + act on.""" + counts = report["counts"] + print(f"Prod tag: {report['prod_tag']}") + print(f"Mode: {'APPLY (mutating)' if report['applied'] else 'dry-run (no mutation)'}") + print( + f"shipped={counts['shipped']} pending={counts['pending']} " + f"triage={counts['triage']} (total Deploy Ready stories seen: {counts['total']})" + ) + print() + + print(f"--- shipped ({len(report['shipped'])}) ---") + for s in report["shipped"]: + if not report["applied"]: + status = "would transition" + elif s.get("transitioned"): + status = "transitioned" + else: + status = f"FAILED: {s.get('transition_error')}" + print(f" {s['id']} {s.get('name', '')!r} via PR(s) {s.get('shipped_via_prs')} [{status}]") + + print(f"--- pending ({len(report['pending'])}) ---") + for s in report["pending"]: + print(f" {s['id']} {s.get('name', '')!r} qualifying PR(s) {s.get('qualifying_prs')} not yet in prod") + + print(f"--- triage ({len(report['triage'])}) ---") + for s in report["triage"]: + if s["reason"] == "non_standard_workflow_or_state": + print( + f" {s['id']} {s.get('name', '')!r} reason=non_standard_workflow_or_state " + f"workflow_id={s.get('workflow_id')} workflow_state_id={s.get('workflow_state_id')}" + ) + else: + print( + f" {s['id']} {s.get('name', '')!r} reason=no_qualifying_pr " + f"linked_pr_numbers={s.get('linked_pr_numbers')}" + ) + + +def build_arg_parser(): + parser = argparse.ArgumentParser( + description="Reconcile Deploy Ready Shortcut stories against what has actually reached prod.", + ) + parser.add_argument( + "--apply", action="store_true", + help="Actually transition qualifying stories Deploy Ready -> Done. Without this, " + "the sweep only classifies and reports; nothing is mutated.", + ) + parser.add_argument( + "--dry-run", action="store_true", + help="Explicit no-op flag for symmetry with mark_stories_deployed.py: classify and " + "report, mutate nothing. This is already the default without --apply, and " + "always wins if both are passed.", + ) + parser.add_argument( + "--prod-tag", + help="Prod tag to check ancestry against (default: newest 'prod/*' tag by creation date).", + ) + parser.add_argument("--repo", default=DEFAULT_REPO, help=f"GitHub repo for `gh pr view` lookups, default {DEFAULT_REPO}") + parser.add_argument( + "--target-branch", default=DEFAULT_TARGET_BRANCH, + help=f"Required PR target branch for a linked PR to count as shipping evidence (guard #2), " + f"default {DEFAULT_TARGET_BRANCH!r}", + ) + parser.add_argument("--out", help="Write the machine-readable JSON report to this path") + parser.add_argument("--max-workers", type=int, default=8) + return parser + + +def main(): + args = build_arg_parser().parse_args() + + token = os.environ.get("SHORTCUT_API_TOKEN") + if not token: + die( + "SHORTCUT_API_TOKEN is not set. Required even for --dry-run: enumerating Deploy " + "Ready stories and resolving their linked PRs both come from the Shortcut API, " + "not from git." + ) + + prod_tag = args.prod_tag or resolve_default_prod_tag() + + try: + stories = search_deploy_ready_stories(token) + except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError, OSError) as e: + die(f"Failed to enumerate Deploy Ready stories from Shortcut: {e}") + + triage, candidates = classify_stories(stories, SEFARIA_PROJECT_REPO_ID, args.target_branch) + + all_qualifying_pr_numbers = sorted( + {pr["number"] for _, prs in candidates for pr in prs if pr.get("number") is not None} + ) + oid_by_pr = fetch_merge_oids(all_qualifying_pr_numbers, args.repo, args.max_workers) + + shipped, pending = classify_candidates(candidates, oid_by_pr, prod_tag) + + # --dry-run always wins over --apply if somehow both are passed -- a + # bulk mutation of shared team state should never be one flag-typo away + # from firing. + apply_mutations = args.apply and not args.dry_run + + failed_transitions = [] + if apply_mutations: + transitioned_ids, failed_transitions = transition_stories( + [s["id"] for s in shipped], DONE_STATE_ID, token, args.max_workers + ) + transitioned_set = set(transitioned_ids) + failed_by_id = dict(failed_transitions) + for s in shipped: + s["transitioned"] = s["id"] in transitioned_set + if s["id"] in failed_by_id: + s["transition_error"] = failed_by_id[s["id"]] + else: + for s in shipped: + s["transitioned"] = None # not attempted -- dry-run + + report = { + "prod_tag": prod_tag, + "applied": apply_mutations, + "counts": { + "total": len(stories), + "shipped": len(shipped), + "pending": len(pending), + "triage": len(triage), + }, + "shipped": shipped, + "pending": pending, + "triage": triage, + } + + print_summary(report) + + if args.out: + with open(args.out, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2, ensure_ascii=False) + f.write("\n") + + if failed_transitions: + warn( + f"{len(failed_transitions)} stor{'y' if len(failed_transitions) == 1 else 'ies'} " + f"failed to transition: {failed_transitions}" + ) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/build/ci/shipped_stories.py b/build/ci/shipped_stories.py index 232c13f121..6d315df74e 100755 --- a/build/ci/shipped_stories.py +++ b/build/ci/shipped_stories.py @@ -4,18 +4,39 @@ Walks the git tree between two prod tags (or an explicit commit range), extracts Shortcut (SC) story ids from commit subjects and, for commits that -reference a merged PR, from that PR's branch name too. Revert commits -(`Revert "..."`, `Revert: ...`, `revert(...)`) never contribute story ids to -the shipped set; their suppressed ids are surfaced separately in -`reverted_commits` instead of being silently dropped. Optionally hydrates -each id via the Shortcut API (id, name, description, url, workflow id, -workflow state, story type) when SHORTCUT_API_TOKEN is set — `workflow_id` -is included because a workflow's Done state id is not universal across -Shortcut workflows, and downstream tooling (mark_stories_deployed.py) needs -it to tell "different workflow" apart from "different state". Emits a -single JSON document that downstream tooling (the sefaria-release-notes -skill, mark_stories_deployed.py) consumes — this script never writes prose -and never mutates a Shortcut story. +reference a merged PR, from that PR's branch name too. As a THIRD discovery +source, a commit whose PR carries no sc-NNNNN id in either place (subject or +branch name) is looked up against Shortcut's own PR<->story link +(`search/stories?query=pr:`) when SHORTCUT_API_TOKEN is set -- git text is +not the only place a story/PR link can live; a story can be linked to a PR +from the Shortcut UI without the PR's branch ever mentioning a story code. +This fallback is NOT run at all for a commit whose subject is auto-generated +merge/branch-sync noise (NOISE_PATTERN) or whose PR's own head branch is a +long-lived environment branch (master/preprod/prod) -- both are shapes a +promotion PR takes, and a promotion PR resolves via `pr:` to a real story +just as readily as that story's actual feature PR does, while proving +nothing about whether that story's own change shipped (verified live: PRs +whose head branch was `preprod` or `master` each resolved to a real story +this way). The single search result is also re-checked against the SAME +three PR-level guards `reconcile_deploy_ready.py`'s org-wide sweep applies +(merged / Sefaria-Project repo / target branch master; shared via +shortcut_pr_guards.py so the two scripts cannot silently drift apart) before +its id is adopted -- a match that fails those guards is a warn-and-skip, not +a fallback of last resort. The id is adopted ONLY when the search resolves +to EXACTLY one story AND that story's PR passes those guards; ids recovered +this way are also surfaced separately in `stories_from_shortcut_pr_link` so +a report can say what only Shortcut knew. +Revert commits (`Revert "..."`, `Revert: ...`, `revert(...)`) never +contribute story ids to the shipped set; their suppressed ids are surfaced +separately in `reverted_commits` instead of being silently dropped. +Optionally hydrates each id via the Shortcut API (id, name, description, +url, workflow id, workflow state, story type) when SHORTCUT_API_TOKEN is +set — `workflow_id` is included because a workflow's Done state id is not +universal across Shortcut workflows, and downstream tooling +(mark_stories_deployed.py) needs it to tell "different workflow" apart from +"different state". Emits a single JSON document that downstream tooling +(the sefaria-release-notes skill, mark_stories_deployed.py) consumes — this +script never writes prose and never mutates a Shortcut story. Usage: python3 shipped_stories.py --version 6.111.0-prod.2 [--out shipped-stories.json] [--repo Sefaria/Sefaria-Project] @@ -31,7 +52,8 @@ succeed — a failing lookup for one PR is logged to stderr and skipped. SHORTCUT_API_TOKEN is optional; without it, story ids are still emitted but `stories` is empty and `unresolved_story_ids` is not populated (hydration -was never attempted, which is a different case from a failed lookup). +was never attempted, which is a different case from a failed lookup), and +the RC1 PR-link fallback above is skipped entirely (git-text discovery only). All ids shown in this file's docstring and comments (e.g. story id 11111) are placeholders, not real Shortcut story ids. @@ -45,8 +67,29 @@ import subprocess import sys import urllib.error +import urllib.parse import urllib.request +# The PR-level shipping-evidence guards (merged / right repo / right target +# branch) are shared with reconcile_deploy_ready.py's org-wide sweep -- see +# shortcut_pr_guards.py's own docstring for why a single shared +# implementation matters (a promotion PR is exactly as good at fooling +# either script). build/ci is not a package (see tests/conftest.py), but a +# plain sibling-module import works both when this file is run directly +# (python3 puts its own directory on sys.path[0]) and under pytest (the +# test conftest adds build/ci to sys.path the same way). +import shortcut_pr_guards + +# Long-lived environment branches. A PR whose HEAD branch is one of these is +# a promotion/branch-sync PR (preprod -> prod, master -> preprod, ...), not +# a real feature PR -- verified live to resolve via the RC1 PR-link fallback +# below to a real story despite proving nothing about whether that story's +# own change shipped. Checked against the PR's *head* branch (the same +# `branch` value already resolved via fetch_pr_branch for story-id +# extraction), not its target branch -- shortcut_pr_guards' target-branch +# guard covers that side separately. +LONG_LIVED_ENV_BRANCHES = frozenset({"master", "preprod", "prod"}) + # Shortcut (SC) story id patterns recognized in a commit subject or a PR # branch name. Kept intentionally short: `\bsc[-_](\d+)\b` (pattern 1) has a # TRAILING word boundary, so it already matches "sc-N"/"sc_N" wrapped in any @@ -308,6 +351,100 @@ def fetch_story(story_id, token): } +def fetch_story_by_pr_link(pr_number, token): + """Look up the single Shortcut story linked to a merged PR via + Shortcut's own PR<->story association (RC1 in the incident writeup): + `GET search/stories?query=pr:`. This is a fallback ONLY for a commit + whose subject and PR branch name both carried no sc-NNNNN id -- git text + is not the only place the link can live; a story can be attached to a PR + from the Shortcut UI with no story code ever appearing in the branch + name (verified case: a PR branched as + `feature/prod-rollout-slack-release-notes`, no story code at all, that + Shortcut still knew was linked to a real story). + + `branch:"..."` and `pull-request:N` search operators do NOT resolve this + -- only `pr:N` does -- so that's the only query shape used here. + + Adopts the id ONLY when the search returns EXACTLY one story AND that + story's OWN linked-PR entry for this exact PR number passes the same + three PR-level guards reconcile_deploy_ready.py's sweep applies (merged + / Sefaria-Project repo / target branch master -- see + shortcut_pr_guards.py). That second check matters because a bare + `pr:` match only proves Shortcut linked SOME story to this PR + number -- not that this PR is real shipping evidence for it. A + promotion PR (head branch `master`/`preprod`/`prod`) resolves via this + same search to a real story just as readily as that story's actual + feature PR does; without re-checking the guards here, that promotion + PR would get silently adopted as if it were proof the story shipped + (verified live). More than one search result is ambiguous (which story + is "the" story for this PR?) and is a warn-and-skip, never a guess. + Any lookup failure (network, HTTP error, bad JSON) is also a + warn-and-skip, matching fetch_story's error posture immediately above + -- this must never abort the run. + """ + query = urllib.parse.quote(f"pr:{pr_number}") + url = f"{SHORTCUT_API_BASE}/search/stories?query={query}" + req = urllib.request.Request(url, headers={"Shortcut-Token": token, "Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + body = resp.read().decode("utf-8") + data = json.loads(body) + except urllib.error.HTTPError as e: + warn(f"Shortcut PR-link lookup for PR #{pr_number} failed: HTTP {e.code} {e.reason}") + return pr_number, None + except Exception as e: # noqa: BLE001 - a lookup failure must never abort the run + warn(f"Shortcut PR-link lookup for PR #{pr_number} failed: {e}") + return pr_number, None + + results = data.get("data", []) + total = data.get("total", len(results)) + if not results: + return pr_number, None + if total > 1 or len(results) > 1: + warn(f"Shortcut PR-link lookup for PR #{pr_number} was ambiguous ({total} stories); skipping.") + return pr_number, None + + story = results[0] + story_id = story.get("id") + if story_id is None: + return pr_number, None + + try: + pr_number_int = int(pr_number) + except (TypeError, ValueError): + pr_number_int = None + + matching_pr = next( + (pr for pr in shortcut_pr_guards.gather_linked_prs(story) if pr.get("number") == pr_number_int), + None, + ) + if matching_pr is None or not shortcut_pr_guards.passes_pr_guards(matching_pr): + warn( + f"Shortcut PR-link lookup for PR #{pr_number} resolved to story {story_id}, but " + "that PR does not pass the shipping-evidence guards (merged / Sefaria-Project " + "repo / target branch master) -- e.g. a promotion or branch-sync merge rather " + "than the real feature PR. Skipping." + ) + return pr_number, None + + return pr_number, str(story_id) + + +def fetch_stories_by_pr(pr_numbers, token, max_workers=8): + """Batch-resolve fetch_story_by_pr_link across PRs concurrently, the + same pattern fetch_branches and hydrate_stories already use below.""" + story_id_by_pr = {} + if not pr_numbers: + return story_id_by_pr + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = [executor.submit(fetch_story_by_pr_link, n, token) for n in pr_numbers] + for future in concurrent.futures.as_completed(futures): + pr_number, story_id = future.result() + if story_id: + story_id_by_pr[pr_number] = story_id + return story_id_by_pr + + def hydrate_stories(story_ids, token, max_workers=8): stories = [] unresolved = [] @@ -386,6 +523,12 @@ def main(): branch_by_pr = fetch_branches(sorted(pr_numbers), args.repo) + # Read once, up front: used both for the RC1 PR-link fallback right + # below and for hydration later in main(). A single token means both a + # missing token and an unreachable Shortcut behave identically in both + # places -- degrade gracefully, never abort the run. + token = os.environ.get("SHORTCUT_API_TOKEN") + # Resolve each commit's full story_ids (subject ∪ its PR branch name) # once, up front -- both the per-commit `commits[]` output and the # aggregate shipped-set logic below read from this. @@ -394,6 +537,55 @@ def main(): c["branch"] = branch c["story_ids"] = c["subject_story_ids"] | extract_story_ids(branch) + # THIRD discovery source (RC1): for any commit that has a PR number but + # STILL resolved to no story id from subject+branch, ask Shortcut + # itself whether that PR is linked to a story. Done at this same + # resolution point -- before carrying_indices_by_id, reverted_commits, + # commits_without_story, or the aggregate shipped set are computed -- + # so every one of those downstream consumers sees the recovered id as + # if it had always been there, with no separate code path to keep in + # sync. + # + # A commit is eligible for this lookup only if, in addition to "has a PR + # number but no story id yet": its subject isn't auto-generated + # merge/branch-sync noise (NOISE_PATTERN already excludes exactly this + # shape from commits_without_story for the same reason -- reused here + # rather than inventing a second notion of "not a real feature commit"), + # and its PR's own head branch isn't a long-lived environment branch + # (LONG_LIVED_ENV_BRANCHES). Both are shapes a promotion PR takes, and a + # promotion PR resolves via `pr:` to a real story just as readily as + # that story's actual feature PR does -- verified live -- so both are + # filtered out here, BEFORE ever calling Shortcut, rather than relying + # solely on fetch_story_by_pr_link's own re-check of the PR itself. + def _eligible_for_pr_link_fallback(c): + return ( + c["pr_number"] + and not c["story_ids"] + and not NOISE_PATTERN.search(c["subject"]) + and c["branch"] not in LONG_LIVED_ENV_BRANCHES + ) + + stories_from_shortcut_pr_link = set() + prs_needing_shortcut_lookup = sorted( + {c["pr_number"] for c in parsed_commits if _eligible_for_pr_link_fallback(c)}, + key=int, + ) + if prs_needing_shortcut_lookup: + if token: + story_id_by_pr = fetch_stories_by_pr(prs_needing_shortcut_lookup, token) + for c in parsed_commits: + if _eligible_for_pr_link_fallback(c): + sid = story_id_by_pr.get(c["pr_number"]) + if sid: + c["story_ids"] = {sid} + stories_from_shortcut_pr_link.add(sid) + else: + warn( + f"SHORTCUT_API_TOKEN is not set; skipping the Shortcut PR-link fallback " + f"lookup for {len(prs_needing_shortcut_lookup)} commit(s) whose PR carries " + "no sc-NNNNN id in its subject or branch name." + ) + # Track, per story id, which NON-revert commit indices carry it. This is # what lets a revert exclude ONLY the specific original commit it quotes # from the shipped set -- not every commit that happens to share that id @@ -450,7 +642,6 @@ def main(): if not c["is_revert"] and not c["story_ids"] and not NOISE_PATTERN.search(c["subject"]): commits_without_story.append(c["subject"]) - token = os.environ.get("SHORTCUT_API_TOKEN") hydrated = bool(token) if token: stories, unresolved_story_ids = hydrate_stories(sorted(all_story_ids, key=int), token) @@ -472,6 +663,12 @@ def main(): "commits_without_story": commits_without_story, "reverted_commits": reverted_commits, "story_ids": sorted(all_story_ids, key=int), + # Ids adopted ONLY via the RC1 Shortcut PR-link fallback above -- + # i.e. story ids git text alone (subject + branch name) never + # revealed. Every id here is also already included in "story_ids" + # (and, if hydration succeeded, in "stories"); this list exists so + # a report can call out what only Shortcut knew. + "stories_from_shortcut_pr_link": sorted(stories_from_shortcut_pr_link, key=int), "stories": stories, # False when SHORTCUT_API_TOKEN was absent, so `stories` being empty # means "never looked up" rather than "looked up and found nothing". diff --git a/build/ci/shortcut_pr_guards.py b/build/ci/shortcut_pr_guards.py new file mode 100644 index 0000000000..89c67159e4 --- /dev/null +++ b/build/ci/shortcut_pr_guards.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +""" +Shared "does this linked PR count as shipping evidence" guards. + +Both `shipped_stories.py` (RC1's PR<->story link fallback) and +`reconcile_deploy_ready.py` (RC2's org-wide Deploy Ready sweep) need to +answer the exact same question about a PR that Shortcut says is linked to a +story: does this PR actually prove that story's change reached prod? A +linked PR is NOT automatically that evidence -- three guards apply, and +both scripts must apply the SAME three guards or they will silently drift +apart (verified live: a promotion PR, e.g. head branch `preprod` merging +into `master`, or `master` merging into `preprod`, resolves via +`search/stories?query=pr:` to a real story just as readily as that +story's actual feature PR does -- but proves nothing about whether that +story's own change shipped). + + 1. `merged` must be true. An open or closed-without-merging PR is not + evidence anything shipped. + 2. `repository_id` must be Sefaria-Project's own (500000103). A story can + link a PR from a different repo; resolving that PR number against + Sefaria-Project instead would find an unrelated (often much older) PR + that happens to share the number. + 3. `target_branch_name` must be "master". A promotion PR (preprod -> + prod, or master -> preprod) merges constantly and proves nothing + about whether a given story's own change reached prod -- it must + never be treated as interchangeable with the real feature PR. + +This module holds the shared implementation so there is exactly one place +these guards live; single-source-of-truth, not two parallel copies that a +future edit only remembers to update in one of them. Stdlib only -- no +third-party dependencies, matching both callers' dependency posture. + +All story/PR ids in this file's docstring and comments (e.g. 500000103, +which is Sefaria-Project's real repository id, not a story id, and is not +covered by the "no real Shortcut ids" convention the two callers document) +are either placeholders or non-story ids -- see shipped_stories.py's and +reconcile_deploy_ready.py's own docstrings for that convention. +""" + +# Sefaria-Project's own Shortcut repository id. Not a story id -- see the +# module docstring's placeholder-id note. +SEFARIA_PROJECT_REPO_ID = 500000103 + +DEFAULT_TARGET_BRANCH = "master" + + +def passes_pr_guards(pr, repo_id=SEFARIA_PROJECT_REPO_ID, target_branch=DEFAULT_TARGET_BRANCH): + """True if a single linked-PR object (a Shortcut `pull-request` entity, + as found in a story's `pull_requests` or `branches[*].pull_requests`) + counts as evidence that a story's change reached prod: merged, against + the right repo, targeting the right branch. See the module docstring + for why each of the three checks exists.""" + return ( + pr.get("merged") is True + and pr.get("repository_id") == repo_id + and pr.get("target_branch_name") == target_branch + ) + + +def qualifying_prs(prs, repo_id=SEFARIA_PROJECT_REPO_ID, target_branch=DEFAULT_TARGET_BRANCH): + """Filter a list of linked-PR objects down to the ones that pass + passes_pr_guards.""" + return [pr for pr in prs if passes_pr_guards(pr, repo_id=repo_id, target_branch=target_branch)] + + +def gather_linked_prs(story): + """Collect every PR linked to a Shortcut story from BOTH + `story.pull_requests` and `story.branches[*].pull_requests` -- Shortcut + duplicates the same PR object in both places, and depending on how/when + a branch or PR was linked, either one can be the only place a given PR + shows up. Deduplicated by PR number (top-level `pull_requests` wins on + a tie; the object is identical either way). Returns every linked PR, + guards not yet applied -- see qualifying_prs/passes_pr_guards -- so a + caller can show what was linked at all, not just what passed.""" + seen = set() + prs = [] + for pr in story.get("pull_requests") or []: + number = pr.get("number") + if number is not None and number not in seen: + seen.add(number) + prs.append(pr) + for branch in story.get("branches") or []: + for pr in branch.get("pull_requests") or []: + number = pr.get("number") + if number is not None and number not in seen: + seen.add(number) + prs.append(pr) + return prs diff --git a/build/ci/tests/test_reconcile_deploy_ready.py b/build/ci/tests/test_reconcile_deploy_ready.py new file mode 100644 index 0000000000..3eca7629ff --- /dev/null +++ b/build/ci/tests/test_reconcile_deploy_ready.py @@ -0,0 +1,524 @@ +"""Tests for build/ci/reconcile_deploy_ready.py. + +No network, no real `git`/`gh` calls: anything that would call +urllib.request.urlopen, subprocess.run, or the module's own run_git is +monkeypatched, matching the style of test_shipped_stories.py and +test_mark_stories_deployed.py. + +All story ids used below (11111, 22222, ...) are placeholders, not real +Shortcut story ids. The repo/workflow/state ids (500000103, 500000005, +500000045, 500000010, 500000061) are real Shortcut/GitHub ids, not story +ids, and are not covered by that placeholder rule -- same convention as the +two existing test files. +""" + +import json + +import pytest + +import reconcile_deploy_ready as rdr + +STANDARD_WORKFLOW_ID = 500000005 +DEPLOY_READY_STATE_ID = 500000045 +DONE_STATE_ID = 500000010 +SEFARIA_REPO_ID = 500000103 +OTHER_REPO_ID = 500000124 # some other Shortcut-linked repo, not Sefaria-Project +# A non-Standard workflow (e.g. "Content") that also has some state it +# reports as workflow_state_id -- deliberately NOT 500000045, since that id +# means something else (or nothing) outside the Standard workflow. +OTHER_WORKFLOW_ID = 500000061 +OTHER_WORKFLOW_STATE_ID = 500000900 + + +def _pr(number, merged=True, repository_id=SEFARIA_REPO_ID, target_branch_name="master"): + return { + "number": number, + "merged": merged, + "repository_id": repository_id, + "target_branch_name": target_branch_name, + } + + +def _story(story_id, name="Story", workflow_id=STANDARD_WORKFLOW_ID, workflow_state_id=DEPLOY_READY_STATE_ID, + pull_requests=None, branches=None): + return { + "id": story_id, + "name": name, + "app_url": f"https://app.shortcut.com/org/story/{story_id}", + "workflow_id": workflow_id, + "workflow_state_id": workflow_state_id, + "pull_requests": pull_requests or [], + "branches": branches or [], + } + + +# --- gather_linked_prs: dedup across pull_requests and branches[].pull_requests -- + +def test_gather_linked_prs_dedups_across_both_sources(): + """Shortcut duplicates the same PR object in story.pull_requests AND + story.branches[*].pull_requests -- the same PR number appearing in both + must not be double-counted.""" + pr = _pr(100) + story = _story(11111, pull_requests=[pr], branches=[{"pull_requests": [pr]}]) + prs = rdr.gather_linked_prs(story) + assert [p["number"] for p in prs] == [100] + + +def test_gather_linked_prs_collects_from_both_sources_when_numbers_differ(): + story = _story(11111, pull_requests=[_pr(100)], branches=[{"pull_requests": [_pr(200)]}]) + prs = rdr.gather_linked_prs(story) + assert sorted(p["number"] for p in prs) == [100, 200] + + +def test_gather_linked_prs_empty_story_returns_empty(): + story = _story(11111) + assert rdr.gather_linked_prs(story) == [] + + +def test_gather_linked_prs_ignores_prs_with_no_number(): + story = _story(11111, pull_requests=[{"merged": True}]) + assert rdr.gather_linked_prs(story) == [] + + +# --- qualifying_prs: the three PR-level guards --------------------------- + +def test_qualifying_prs_wrong_repo_guard(): + """Guard #1: a PR linked from a DIFFERENT repo must never count as + evidence a Sefaria-Project story shipped, even if it's merged and + targets 'master' -- resolving it against Sefaria-Project would find an + unrelated PR that happens to share the number.""" + prs = [_pr(224, repository_id=OTHER_REPO_ID)] + assert rdr.qualifying_prs(prs) == [] + + +def test_qualifying_prs_promotion_pr_guard(): + """Guard #2: a PR targeting anything other than 'master' (e.g. a + preprod->prod or master->preprod promotion PR some stories link instead + of the real feature PR) must not qualify.""" + prs = [_pr(3551, target_branch_name="prod")] + assert rdr.qualifying_prs(prs) == [] + prs2 = [_pr(3550, target_branch_name="preprod")] + assert rdr.qualifying_prs(prs2) == [] + + +def test_qualifying_prs_unmerged_pr_guard(): + """Guard #3: an open or closed-without-merging PR proves nothing.""" + prs = [_pr(3397, merged=False)] + assert rdr.qualifying_prs(prs) == [] + + +def test_qualifying_prs_accepts_a_pr_passing_all_three_guards(): + prs = [_pr(3606)] + assert rdr.qualifying_prs(prs) == prs + + +def test_qualifying_prs_filters_mixed_list_keeping_only_the_valid_one(): + prs = [ + _pr(224, repository_id=OTHER_REPO_ID), + _pr(3551, target_branch_name="prod"), + _pr(3397, merged=False), + _pr(3606), + ] + assert [p["number"] for p in rdr.qualifying_prs(prs)] == [3606] + + +def test_qualifying_prs_custom_repo_and_target_branch_args(): + """--repo/--target-branch overrides are threaded through, not hardcoded.""" + prs = [_pr(1, repository_id=999, target_branch_name="main")] + assert rdr.qualifying_prs(prs, repo_id=999, target_branch="main") == prs + assert rdr.qualifying_prs(prs, repo_id=SEFARIA_REPO_ID, target_branch="master") == [] + + +# --- classify_stories: guard #4 (workflow/state) + guard-filtered triage -- + +def test_classify_stories_non_standard_workflow_routes_to_triage_not_transition(): + """Guard #4: a story on a non-Standard workflow must be routed to + triage with its actual workflow/state ids reported -- never treated as + a shipping candidate, since 500000045/500000010 mean nothing there.""" + story = _story(11111, workflow_id=OTHER_WORKFLOW_ID, workflow_state_id=OTHER_WORKFLOW_STATE_ID, + pull_requests=[_pr(1)]) + triage, candidates = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + assert candidates == [] + assert len(triage) == 1 + assert triage[0]["id"] == 11111 + assert triage[0]["reason"] == "non_standard_workflow_or_state" + assert triage[0]["workflow_id"] == OTHER_WORKFLOW_ID + assert triage[0]["workflow_state_id"] == OTHER_WORKFLOW_STATE_ID + + +def test_classify_stories_unexpected_state_within_standard_workflow_routes_to_triage(): + """Guard #4 also keys on the NUMERIC state id, not just the workflow -- + a story on the Standard workflow but at some other state id must not + be treated as Deploy Ready just because it matched the search by name.""" + story = _story(11111, workflow_id=STANDARD_WORKFLOW_ID, workflow_state_id=500000099, + pull_requests=[_pr(1)]) + triage, candidates = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + assert candidates == [] + assert triage[0]["reason"] == "non_standard_workflow_or_state" + + +def test_classify_stories_no_qualifying_pr_routes_to_triage(): + story = _story(11111, pull_requests=[_pr(3397, merged=False)]) + triage, candidates = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + assert candidates == [] + assert triage[0]["reason"] == "no_qualifying_pr" + assert triage[0]["linked_pr_numbers"] == [3397] + + +def test_classify_stories_story_with_no_linked_prs_at_all_routes_to_triage(): + story = _story(11111) + triage, candidates = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + assert triage[0]["reason"] == "no_qualifying_pr" + assert triage[0]["linked_pr_numbers"] == [] + + +def test_classify_stories_qualifying_story_becomes_a_candidate(): + story = _story(11111, pull_requests=[_pr(3606)]) + triage, candidates = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + assert triage == [] + assert len(candidates) == 1 + assert candidates[0][0]["id"] == 11111 + assert [p["number"] for p in candidates[0][1]] == [3606] + + +# --- classify_candidates: ancestor true vs false, unresolved oid --------- + +def test_classify_candidates_ancestor_true_is_shipped(monkeypatch): + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: True) + story = _story(11111, pull_requests=[_pr(3606)]) + candidates = [(story, [_pr(3606)])] + shipped, pending = rdr.classify_candidates(candidates, {3606: "abc123"}, "prod/1.0") + assert pending == [] + assert len(shipped) == 1 + assert shipped[0]["id"] == 11111 + assert shipped[0]["shipped_via_prs"] == [3606] + assert shipped[0]["qualifying_prs"] == [3606] + + +def test_classify_candidates_ancestor_false_is_pending(monkeypatch): + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: False) + story = _story(11111, pull_requests=[_pr(3670)]) + candidates = [(story, [_pr(3670)])] + shipped, pending = rdr.classify_candidates(candidates, {3670: "def456"}, "prod/1.0") + assert shipped == [] + assert len(pending) == 1 + assert pending[0]["id"] == 11111 + assert pending[0]["qualifying_prs"] == [3670] + + +def test_classify_candidates_unresolved_oid_never_counts_as_shipped(monkeypatch): + """A PR whose merge oid never resolved (gh lookup failed) must push a + story toward pending, never toward shipped -- a resolution failure must + never cause a wrong transition.""" + def _boom(oid, tag): + raise AssertionError("is_ancestor_of_prod must not be called for an unresolved oid") + + monkeypatch.setattr(rdr, "is_ancestor_of_prod", _boom) + story = _story(11111, pull_requests=[_pr(3606)]) + candidates = [(story, [_pr(3606)])] + shipped, pending = rdr.classify_candidates(candidates, {}, "prod/1.0") # oid_by_pr empty: unresolved + assert shipped == [] + assert len(pending) == 1 + + +def test_classify_candidates_inconclusive_ancestry_check_never_counts_as_shipped(monkeypatch): + """is_ancestor_of_prod returning None (the check itself couldn't run) + must also never be treated as "in prod".""" + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: None) + story = _story(11111, pull_requests=[_pr(3606)]) + candidates = [(story, [_pr(3606)])] + shipped, pending = rdr.classify_candidates(candidates, {3606: "abc"}, "prod/1.0") + assert shipped == [] + assert len(pending) == 1 + + +def test_classify_candidates_any_qualifying_pr_in_prod_is_enough(monkeypatch): + """A story with two qualifying PRs, only one of which is in prod, still + ships -- 'at least one' per the spec.""" + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: oid == "in-prod-oid") + + story = _story(11111, pull_requests=[_pr(1), _pr(2)]) + candidates = [(story, [_pr(1), _pr(2)])] + oid_by_pr = {1: "not-in-prod-oid", 2: "in-prod-oid"} + shipped, pending = rdr.classify_candidates(candidates, oid_by_pr, "prod/1.0") + assert len(shipped) == 1 + assert shipped[0]["shipped_via_prs"] == [2] + assert shipped[0]["qualifying_prs"] == [1, 2] + + +# --- is_ancestor_of_prod: exit-code interpretation ----------------------- + +def test_is_ancestor_of_prod_true_on_exit_code_0(monkeypatch): + class _Proc: + returncode = 0 + stderr = "" + + monkeypatch.setattr(rdr.subprocess, "run", lambda *a, **k: _Proc()) + assert rdr.is_ancestor_of_prod("abc", "prod/1.0") is True + + +def test_is_ancestor_of_prod_false_on_exit_code_1(monkeypatch): + class _Proc: + returncode = 1 + stderr = "" + + monkeypatch.setattr(rdr.subprocess, "run", lambda *a, **k: _Proc()) + assert rdr.is_ancestor_of_prod("abc", "prod/1.0") is False + + +def test_is_ancestor_of_prod_none_on_other_exit_code(monkeypatch, capsys): + class _Proc: + returncode = 128 + stderr = "fatal: not a valid object name abc" + + monkeypatch.setattr(rdr.subprocess, "run", lambda *a, **k: _Proc()) + assert rdr.is_ancestor_of_prod("abc", "prod/1.0") is None + assert "WARNING" in capsys.readouterr().err + + +# --- search_deploy_ready_stories: pagination ------------------------------ + +class _FakeSearchResponse: + def __init__(self, payload): + self._body = json.dumps(payload).encode("utf-8") + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return self._body + + +def test_search_deploy_ready_stories_paginates_via_next_cursor(monkeypatch): + """Two pages, exactly as verified live (20 stories, 10 per page): must + not assume a single page.""" + pages = [ + {"data": [{"id": 1}, {"id": 2}], "next": "/api/v3/search/stories?query=x&next=CURSOR1"}, + {"data": [{"id": 3}], "next": None}, + ] + calls = [] + + def _fake_urlopen(req, timeout=None): + calls.append(req.full_url) + return _FakeSearchResponse(pages[len(calls) - 1]) + + monkeypatch.setattr(rdr.urllib.request, "urlopen", _fake_urlopen) + stories = rdr.search_deploy_ready_stories("fake-token-for-tests") + assert [s["id"] for s in stories] == [1, 2, 3] + assert len(calls) == 2 + # The second call must follow the `next` cursor, not repeat page one. + assert "CURSOR1" in calls[1] + + +def test_search_deploy_ready_stories_single_page_stops_when_next_is_none(monkeypatch): + def _fake_urlopen(req, timeout=None): + return _FakeSearchResponse({"data": [{"id": 1}], "next": None}) + + monkeypatch.setattr(rdr.urllib.request, "urlopen", _fake_urlopen) + stories = rdr.search_deploy_ready_stories("fake-token-for-tests") + assert [s["id"] for s in stories] == [1] + + +# --- fetch_merge_oids: gh missing must not surface as a raw traceback ---- + +def test_fetch_merge_oids_dies_clearly_when_gh_is_missing(monkeypatch): + def _raise_missing_gh(*args, **kwargs): + raise FileNotFoundError("[Errno 2] No such file or directory: 'gh'") + + monkeypatch.setattr(rdr.subprocess, "run", _raise_missing_gh) + with pytest.raises(SystemExit): + rdr.fetch_merge_oids([3606], "Sefaria/Sefaria-Project") + + +def test_fetch_merge_oids_empty_input_makes_no_calls(monkeypatch): + def _boom(*args, **kwargs): + raise AssertionError("subprocess.run must not be called with no PR numbers") + + monkeypatch.setattr(rdr.subprocess, "run", _boom) + assert rdr.fetch_merge_oids([], "Sefaria/Sefaria-Project") == {} + + +def test_fetch_pr_merge_oid_failed_gh_call_returns_none(monkeypatch, capsys): + class _Proc: + returncode = 1 + stdout = "" + stderr = "PR not found" + + monkeypatch.setattr(rdr.subprocess, "run", lambda *a, **k: _Proc()) + pr_number, oid = rdr.fetch_pr_merge_oid(9999, "Sefaria/Sefaria-Project") + assert oid is None + assert "WARNING" in capsys.readouterr().err + + +# --- End-to-end main(): dry-run is the default and mutates nothing ------- + +def _make_main_env(monkeypatch, tmp_path, stories, prod_tag="prod/1.0", oid_by_pr=None, + ancestor_result=True, argv_extra=None): + """Wire main() end-to-end with every I/O boundary mocked: Shortcut + search, gh merge-commit lookup, and git ancestry -- mirroring + _run_main_with_commits in test_shipped_stories.py.""" + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + monkeypatch.setattr(rdr, "search_deploy_ready_stories", lambda token: stories) + monkeypatch.setattr(rdr, "resolve_default_prod_tag", lambda: prod_tag) + oid_map = oid_by_pr or {} + monkeypatch.setattr( + rdr, "fetch_merge_oids", + lambda pr_numbers, repo, max_workers=8: {n: oid_map[n] for n in pr_numbers if n in oid_map}, + ) + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: ancestor_result) + + argv = ["reconcile_deploy_ready.py"] + (argv_extra or []) + monkeypatch.setattr("sys.argv", argv) + + +def test_main_dry_run_is_the_default_and_never_calls_transition(monkeypatch, tmp_path, capsys): + story = _story(11111, pull_requests=[_pr(3606)]) + _make_main_env(monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True) + + def _boom(*args, **kwargs): + raise AssertionError("transition_story must never be called without --apply") + + monkeypatch.setattr(rdr, "transition_story", _boom) + rdr.main() # must not raise, must not exit non-zero + + out = capsys.readouterr().out + assert "shipped=1" in out + assert "would transition" in out + + +def test_main_apply_transitions_shipped_stories(monkeypatch, tmp_path, capsys): + story = _story(11111, pull_requests=[_pr(3606)]) + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--apply"], + ) + + calls = [] + + def _fake_transition(story_id, done_state_id, token): + calls.append((story_id, done_state_id)) + return story_id, True, None + + monkeypatch.setattr(rdr, "transition_story", _fake_transition) + rdr.main() + + assert calls == [(11111, DONE_STATE_ID)] + + +def test_main_dry_run_flag_overrides_apply(monkeypatch, tmp_path, capsys): + """--dry-run always wins if somehow both --apply and --dry-run are + passed -- a bulk mutation across the whole org must never be one + flag-typo away from firing.""" + story = _story(11111, pull_requests=[_pr(3606)]) + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--apply", "--dry-run"], + ) + + def _boom(*args, **kwargs): + raise AssertionError("transition_story must never be called when --dry-run is also passed") + + monkeypatch.setattr(rdr, "transition_story", _boom) + rdr.main() # must not raise + + +def test_main_writes_out_json_report(monkeypatch, tmp_path): + story_shipped = _story(11111, pull_requests=[_pr(3606)]) + story_triage = _story(22222) # no linked PRs at all + out_path = tmp_path / "report.json" + _make_main_env( + monkeypatch, tmp_path, [story_shipped, story_triage], + oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--out", str(out_path)], + ) + rdr.main() + + report = json.loads(out_path.read_text(encoding="utf-8")) + assert report["counts"] == {"total": 2, "shipped": 1, "pending": 0, "triage": 1} + assert report["applied"] is False + assert [s["id"] for s in report["shipped"]] == [11111] + assert [s["id"] for s in report["triage"]] == [22222] + + +def test_main_pending_bucket_when_qualifying_pr_not_yet_in_prod(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3670)]) + out_path = tmp_path / "report.json" + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3670: "def"}, ancestor_result=False, + argv_extra=["--out", str(out_path)], + ) + rdr.main() + report = json.loads(out_path.read_text(encoding="utf-8")) + assert report["counts"] == {"total": 1, "shipped": 0, "pending": 1, "triage": 0} + assert report["pending"][0]["id"] == 11111 + + +def test_main_missing_token_exits_before_any_shortcut_call(monkeypatch, tmp_path): + monkeypatch.delenv("SHORTCUT_API_TOKEN", raising=False) + + def _boom(*args, **kwargs): + raise AssertionError("no Shortcut call should happen without a token") + + monkeypatch.setattr(rdr, "search_deploy_ready_stories", _boom) + monkeypatch.setattr("sys.argv", ["reconcile_deploy_ready.py"]) + with pytest.raises(SystemExit): + rdr.main() + + +def test_main_failed_transition_exits_non_zero_and_is_reported(monkeypatch, tmp_path, capsys): + """A story that fails to transition must be reported, not swallowed -- + and the process must exit non-zero so an unattended run can't look + successful.""" + story = _story(11111, pull_requests=[_pr(3606)]) + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--apply"], + ) + monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, False, "HTTP 500 Internal Server Error")) + + with pytest.raises(SystemExit) as exc_info: + rdr.main() + assert exc_info.value.code != 0 + err = capsys.readouterr().err + assert "11111" in err + + +def test_main_prod_tag_override_is_used_instead_of_default(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3606)]) + out_path = tmp_path / "report.json" + + def _boom_default_tag(): + raise AssertionError("resolve_default_prod_tag must not be called when --prod-tag is given") + + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + monkeypatch.setattr(rdr, "search_deploy_ready_stories", lambda token: [story]) + monkeypatch.setattr(rdr, "resolve_default_prod_tag", _boom_default_tag) + monkeypatch.setattr(rdr, "fetch_merge_oids", lambda pr_numbers, repo, max_workers=8: {3606: "abc"}) + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: tag == "prod/explicit-tag") + monkeypatch.setattr( + "sys.argv", + ["reconcile_deploy_ready.py", "--prod-tag", "prod/explicit-tag", "--out", str(out_path)], + ) + rdr.main() + report = json.loads(out_path.read_text(encoding="utf-8")) + assert report["prod_tag"] == "prod/explicit-tag" + assert report["counts"]["shipped"] == 1 + + +# --- resolve_default_prod_tag: newest by creation date ------------------- + +def test_resolve_default_prod_tag_picks_newest(monkeypatch): + monkeypatch.setattr( + rdr, "run_git", + lambda args: "prod/2.0+chart.1\nprod/1.0+chart.1\n", + ) + assert rdr.resolve_default_prod_tag() == "prod/2.0+chart.1" + + +def test_resolve_default_prod_tag_dies_with_no_tags(monkeypatch): + monkeypatch.setattr(rdr, "run_git", lambda args: "") + with pytest.raises(SystemExit): + rdr.resolve_default_prod_tag() diff --git a/build/ci/tests/test_shipped_stories.py b/build/ci/tests/test_shipped_stories.py index 49e82900d0..8e0bfccc37 100644 --- a/build/ci/tests/test_shipped_stories.py +++ b/build/ci/tests/test_shipped_stories.py @@ -381,6 +381,155 @@ def test_fetch_story_workflow_id_defaults_to_none_when_absent(monkeypatch): assert data["workflow_id"] is None +# --- fetch_story_by_pr_link: RC1 Shortcut PR<->story fallback ---------- +# (story ids below, e.g. 66666, are placeholders, not real Shortcut ids, +# per repo convention. PR numbers like 3653 are real-shaped GitHub PR +# numbers used only as plausible test fixtures -- PR numbers are not +# Shortcut story ids and are not covered by that convention.) + + +def _story_with_linked_pr(story_id, pr_number, merged=True, repository_id=500000103, target_branch_name="master"): + """Minimal Shortcut search-result story payload carrying ONE linked PR + -- enough for fetch_story_by_pr_link's guard re-check + (shortcut_pr_guards.gather_linked_prs / passes_pr_guards) to find and + evaluate it. repository_id 500000103 is Sefaria-Project's real + Shortcut repository id, not a story id -- exempt from the + placeholder-id convention, same as elsewhere in this file.""" + return { + "id": story_id, + "pull_requests": [{ + "number": int(pr_number), + "merged": merged, + "repository_id": repository_id, + "target_branch_name": target_branch_name, + }], + } + + +def test_fetch_story_by_pr_link_adopts_single_match(monkeypatch): + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse( + {"data": [_story_with_linked_pr(66666, "3653")], "total": 1} + ), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("3653", "fake-token-for-tests") + assert pr_number == "3653" + assert story_id == "66666" + + +def test_fetch_story_by_pr_link_wrong_repo_guard_rejects(monkeypatch, capsys): + """The story IS linked to this PR number, but that PR is against a + DIFFERENT repo -- resolving it against Sefaria-Project would be + exactly the wrong-repo false positive the guard exists to prevent.""" + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse( + {"data": [_story_with_linked_pr(66666, "224", repository_id=500000124)], "total": 1} + ), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("224", "fake-token-for-tests") + assert story_id is None + err = capsys.readouterr().err + assert "does not pass the shipping-evidence guards" in err + assert "224" in err + + +def test_fetch_story_by_pr_link_non_master_target_branch_guard_rejects(monkeypatch, capsys): + """The linked PR merged, against the right repo, but targets `preprod` + (a promotion PR) instead of `master` -- must never be adopted as + evidence a story's own change shipped. This is the exact live failure + case: a promotion PR resolves via `pr:` to a real story just as + readily as that story's actual feature PR does.""" + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse( + {"data": [_story_with_linked_pr(66666, "3698", target_branch_name="preprod")], "total": 1} + ), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("3698", "fake-token-for-tests") + assert story_id is None + assert "does not pass the shipping-evidence guards" in capsys.readouterr().err + + +def test_fetch_story_by_pr_link_unmerged_pr_guard_rejects(monkeypatch, capsys): + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse( + {"data": [_story_with_linked_pr(66666, "3606", merged=False)], "total": 1} + ), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("3606", "fake-token-for-tests") + assert story_id is None + assert "does not pass the shipping-evidence guards" in capsys.readouterr().err + + +def test_fetch_story_by_pr_link_no_match_returns_none_quietly(monkeypatch, capsys): + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse({"data": [], "total": 0}), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("9999", "fake-token-for-tests") + assert story_id is None + # No story linked is an ordinary, expected outcome -- not a warning. + assert capsys.readouterr().err == "" + + +def test_fetch_story_by_pr_link_ambiguous_match_warns_and_skips(monkeypatch, capsys): + """More than one story resolves for the same PR -- never guess which + one is right.""" + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse({"data": [{"id": 1}, {"id": 2}], "total": 2}), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("3677", "fake-token-for-tests") + assert story_id is None + err = capsys.readouterr().err + assert "ambiguous" in err + assert "3677" in err + + +def test_fetch_story_by_pr_link_http_error_does_not_abort(monkeypatch, capsys): + def _boom(req, timeout=None): + raise ss.urllib.error.HTTPError(req.full_url, 500, "Internal Server Error", None, None) + + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + pr_number, story_id = ss.fetch_story_by_pr_link("42", "fake-token-for-tests") + assert story_id is None + assert "WARNING" in capsys.readouterr().err + + +def test_fetch_story_by_pr_link_network_error_does_not_abort(monkeypatch, capsys): + def _boom(req, timeout=None): + raise ss.urllib.error.URLError("network is down") + + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + pr_number, story_id = ss.fetch_story_by_pr_link("42", "fake-token-for-tests") + assert story_id is None + assert "WARNING" in capsys.readouterr().err + + +def test_fetch_stories_by_pr_skips_unresolved(monkeypatch): + monkeypatch.setattr(ss, "fetch_story_by_pr_link", lambda pr, token: (pr, None)) + assert ss.fetch_stories_by_pr(["1", "2"], "fake-token-for-tests") == {} + + +def test_fetch_stories_by_pr_collects_only_resolved_ids(monkeypatch): + def _fake(pr, token): + return (pr, "66666") if pr == "3653" else (pr, None) + + monkeypatch.setattr(ss, "fetch_story_by_pr_link", _fake) + assert ss.fetch_stories_by_pr(["1", "3653"], "fake-token-for-tests") == {"3653": "66666"} + + +def test_fetch_stories_by_pr_empty_input_makes_no_calls(monkeypatch): + def _boom(*args, **kwargs): + raise AssertionError("fetch_story_by_pr_link must not be called with no PR numbers") + + monkeypatch.setattr(ss, "fetch_story_by_pr_link", _boom) + assert ss.fetch_stories_by_pr([], "fake-token-for-tests") == {} + + # --- main(): revert commits are suppressed from story_ids and reported -- def _run_main_with_commits(monkeypatch, tmp_path, commit_subjects): @@ -556,6 +705,230 @@ def _fake_run_git(args): assert data["release_date"] == "2026-08-31T07:17:36Z" +# --- main(): the RC1 Shortcut PR-link fallback end-to-end --------------- + +def test_main_recovers_story_id_via_shortcut_pr_link_fallback(monkeypatch, tmp_path): + """Real-shaped regression case: PR #3653 merged with branch name + `feature/prod-rollout-slack-release-notes` -- no sc-NNNNN code anywhere + in the subject or the branch -- but Shortcut's own PR<->story link knew + it was story 66666. Both git-only discovery sources come up empty; only + the RC1 fallback recovers it, and it must be reported in + stories_from_shortcut_pr_link specifically (not just story_ids).""" + + def _fake_run_git(args): + if args[0] == "log": + return "Announce prod releases from Argo's post-promotion analysis (#3653)\n" + if args[0] == "for-each-ref": + return "" + raise AssertionError(f"unexpected git call: {args!r}") + + monkeypatch.setattr(ss, "run_git", _fake_run_git) + monkeypatch.setattr( + ss, "fetch_pr_branch", + lambda pr_number, repo: (pr_number, "feature/prod-rollout-slack-release-notes"), + ) + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + + def _fake_urlopen(req, timeout=None): + if "search/stories" in req.full_url: + return _FakeShortcutResponse({"data": [_story_with_linked_pr(66666, "3653")], "total": 1}) + # hydrate_stories' fetch_story call for the id the fallback recovered. + return _FakeShortcutResponse({ + "id": 66666, "name": "Story", "description": "", "app_url": "u", + "workflow_id": 500000005, "workflow_state_id": 500000045, "story_type": "feature", + }) + + monkeypatch.setattr(ss.urllib.request, "urlopen", _fake_urlopen) + + out_path = tmp_path / "shipped-stories.json" + monkeypatch.setattr( + "sys.argv", + ["shipped_stories.py", "--range", "prev-tag..cur-tag", "--out", str(out_path)], + ) + ss.main() + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["stories_from_shortcut_pr_link"] == ["66666"] + assert data["story_ids"] == ["66666"] + + +# --- main(): promotion PRs must never be adopted via the RC1 fallback --- +# --- (regression for the live false-positive the coordinator found) ----- + +def test_main_pr_link_fallback_skips_bare_merge_commit_promotion_pr(monkeypatch, tmp_path): + """Real live evidence: a bare 'Merge pull request #N from + Sefaria/preprod' promotion-merge subject resolves via `pr:` to a + real story (exactly as readily as a real feature PR would) but proves + nothing about whether that story shipped. NOISE_PATTERN already + recognizes this exact subject shape (it's excluded from + commits_without_story for the same underlying reason) -- reused here + to filter it out of the fallback candidate set BEFORE any network call, + not merely relying on fetch_story_by_pr_link's own guard re-check.""" + + def _fake_run_git(args): + if args[0] == "log": + return "Merge pull request #3698 from Sefaria/preprod\n" + if args[0] == "for-each-ref": + return "" + raise AssertionError(f"unexpected git call: {args!r}") + + def _boom(*args, **kwargs): + raise AssertionError( + "urlopen must never be called for a commit whose subject is NOISE_PATTERN noise " + "-- the pre-filter must exclude it before any network call is attempted" + ) + + monkeypatch.setattr(ss, "run_git", _fake_run_git) + # A bare merge-commit subject like this has no `(#N)` form for + # fetch_pr_branch to key off of via extract_pr_number's parenthesized + # pattern -- MERGE_PR_PATTERN resolves the PR number from the subject + # itself, so the branch lookup below is irrelevant to reaching pr_number + # but is still stubbed defensively. + monkeypatch.setattr(ss, "fetch_pr_branch", lambda pr_number, repo: (pr_number, None)) + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + + out_path = tmp_path / "shipped-stories.json" + monkeypatch.setattr( + "sys.argv", + ["shipped_stories.py", "--range", "prev-tag..cur-tag", "--out", str(out_path)], + ) + ss.main() + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["stories_from_shortcut_pr_link"] == [] + assert data["story_ids"] == [] + + +def test_main_pr_link_fallback_skips_long_lived_env_head_branch(monkeypatch, tmp_path): + """Second layer of the same guard: even for a commit whose subject does + NOT match NOISE_PATTERN (e.g. a squash-merged promotion PR with an + ordinary-looking subject), a PR whose own HEAD branch is a long-lived + environment branch (master/preprod/prod) must still never reach the + Shortcut lookup -- that's the shape a promotion PR takes regardless of + how its merge commit's subject happens to read.""" + + def _fake_run_git(args): + if args[0] == "log": + return "chore: promote build (#3699)\n" + if args[0] == "for-each-ref": + return "" + raise AssertionError(f"unexpected git call: {args!r}") + + def _boom(*args, **kwargs): + raise AssertionError( + "urlopen must never be called for a commit whose PR's head branch is a " + "long-lived environment branch" + ) + + monkeypatch.setattr(ss, "run_git", _fake_run_git) + monkeypatch.setattr(ss, "fetch_pr_branch", lambda pr_number, repo: (pr_number, "master")) + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + + out_path = tmp_path / "shipped-stories.json" + monkeypatch.setattr( + "sys.argv", + ["shipped_stories.py", "--range", "prev-tag..cur-tag", "--out", str(out_path)], + ) + ss.main() + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["stories_from_shortcut_pr_link"] == [] + assert data["story_ids"] == [] + + +def test_main_pr_link_fallback_not_triggered_when_subject_already_has_story_id(monkeypatch, tmp_path): + """A commit whose subject already carries a story id must never trigger + the fallback lookup at all -- it has nothing missing to recover.""" + + def _fake_run_git(args): + if args[0] == "log": + return "fix(sc-13): a change (#42)\n" + if args[0] == "for-each-ref": + return "" + raise AssertionError(f"unexpected git call: {args!r}") + + def _boom(*args, **kwargs): + raise AssertionError("urlopen must not be called when the commit already has a story id") + + monkeypatch.setattr(ss, "run_git", _fake_run_git) + monkeypatch.setattr(ss, "fetch_pr_branch", lambda pr_number, repo: (pr_number, None)) + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + + out_path = tmp_path / "shipped-stories.json" + monkeypatch.setattr( + "sys.argv", + ["shipped_stories.py", "--range", "prev-tag..cur-tag", "--out", str(out_path)], + ) + ss.main() + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["stories_from_shortcut_pr_link"] == [] + assert data["story_ids"] == ["13"] + + +def test_main_pr_link_fallback_skipped_without_token(monkeypatch, tmp_path, capsys): + """SHORTCUT_API_TOKEN gate: without a token, the fallback must be + skipped entirely (no network call attempted) and must not crash the + run -- it degrades to git-only discovery.""" + + def _fake_run_git(args): + if args[0] == "log": + return "Some subject (#42)\n" + if args[0] == "for-each-ref": + return "" + raise AssertionError(f"unexpected git call: {args!r}") + + def _boom(*args, **kwargs): + raise AssertionError("urlopen must never be called without a token") + + monkeypatch.setattr(ss, "run_git", _fake_run_git) + monkeypatch.setattr(ss, "fetch_pr_branch", lambda pr_number, repo: (pr_number, "no-story-code-branch")) + monkeypatch.delenv("SHORTCUT_API_TOKEN", raising=False) + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + + out_path = tmp_path / "shipped-stories.json" + monkeypatch.setattr( + "sys.argv", + ["shipped_stories.py", "--range", "prev-tag..cur-tag", "--out", str(out_path)], + ) + ss.main() + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["stories_from_shortcut_pr_link"] == [] + assert data["story_ids"] == [] + assert "SHORTCUT_API_TOKEN is not set" in capsys.readouterr().err + + +def test_main_pr_link_fallback_lookup_failure_does_not_abort_run(monkeypatch, tmp_path): + """A Shortcut PR-link lookup failure (network error, HTTP error, ...) + must be a warn-and-skip, never something that aborts shipped_stories.py + entirely -- the whole point of finding/fixing RC1 must not introduce a + new way for the script to die.""" + + def _fake_run_git(args): + if args[0] == "log": + return "Some subject (#42)\n" + if args[0] == "for-each-ref": + return "" + raise AssertionError(f"unexpected git call: {args!r}") + + def _boom(req, timeout=None): + raise ss.urllib.error.URLError("network is down") + + monkeypatch.setattr(ss, "run_git", _fake_run_git) + monkeypatch.setattr(ss, "fetch_pr_branch", lambda pr_number, repo: (pr_number, "no-story-code-branch")) + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + + out_path = tmp_path / "shipped-stories.json" + monkeypatch.setattr( + "sys.argv", + ["shipped_stories.py", "--range", "prev-tag..cur-tag", "--out", str(out_path)], + ) + ss.main() # must return normally, not raise + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["stories_from_shortcut_pr_link"] == [] + assert data["story_ids"] == [] + + # --- fetch_branches: a missing `gh` binary must not surface as a raw ------ # --- traceback (finding #11) ---------------------------------------------- From c319635a433aaeb748b6e2b6d9aba8b310d6f8a6 Mon Sep 17 00:00:00 2001 From: Yotam Fromm Date: Mon, 7 Sep 2026 09:27:33 +0300 Subject: [PATCH 2/6] fix(ci): write back which release shipped a story as a Shortcut comment mark_stories_deployed.py and reconcile_deploy_ready.py only ever wrote a story's workflow state; nothing recorded which release actually carried it. Both now post a short comment via the shared shortcut_comment.py immediately after a real transition. mark_stories_deployed.py names its own release directly (it's reading that release's shipped-stories.json); reconcile_deploy_ready.py instead derives the TRUE shipping release from `git tag --list --contains ` and degrades honestly if that can't be resolved, since naming the current prod tag for a backfilled story would be the same "old features shipped today" mistake this pipeline's other separations already guard against. Comments are never posted in dry-run/no-apply mode or with the new --no-comment flag, and a comment failure is reported but never fails the run or rolls back the transition. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KJqBksQqzHYs3F54Y4tJRX --- build/ci/README-prod-release-notes.md | 89 +++++- build/ci/mark_stories_deployed.py | 118 ++++++++ build/ci/reconcile_deploy_ready.py | 144 ++++++++++ build/ci/shortcut_comment.py | 66 +++++ build/ci/tests/test_mark_stories_deployed.py | 216 ++++++++++++++- build/ci/tests/test_reconcile_deploy_ready.py | 256 +++++++++++++++++- 6 files changed, 873 insertions(+), 16 deletions(-) create mode 100644 build/ci/shortcut_comment.py diff --git a/build/ci/README-prod-release-notes.md b/build/ci/README-prod-release-notes.md index 34ad9c36f0..7359e5a1b6 100644 --- a/build/ci/README-prod-release-notes.md +++ b/build/ci/README-prod-release-notes.md @@ -17,18 +17,24 @@ Argo post-promotion analysis (prod) link, hydrates story details -> build/ci/mark_stories_deployed.py — moves each shipped story Deploy Ready -> Done via the - Shortcut API. A failure here (missing - token, API error, nothing to move) is - logged and Slack-alerted but never - blocks the steps below. + Shortcut API, then posts a write-back + comment naming this release. A + failure here (missing token, API + error, nothing to move) is logged and + Slack-alerted but never blocks the + steps below. -> build/ci/reconcile_deploy_ready.py — separately, sweeps EVERY non-archived Deploy Ready story org-wide (not just this release's commit range) and transitions any - whose linked PR already reached prod - — see "Reconciliation sweep" below. - Its output NEVER reaches the two - steps that follow. + whose linked PR already reached prod, + then posts its own write-back comment + naming the release that ACTUALLY + shipped it (never this one) — see + "Reconciliation sweep" and "Write-back + release comment" below. Its output + NEVER reaches the two steps that + follow. -> sefaria-release-notes skill — reads shipped-stories.json, writes prose only -> scripts/post_to_slack.py — posts both files to Slack @@ -194,6 +200,73 @@ enforces the separation. A failure here is warned/Slack-alerted the same way a `mark_stories_deployed.py` failure is, and never blocks release-notes generation or posting. +## Write-back release comment + +Until this feature, the pipeline only ever WROTE a story's workflow state +(the `PUT` that moves it Deploy Ready -> Done) — nothing recorded WHICH +release actually carried a story, so a person reading it in Shortcut could +see it became Done but had no way to tell what shipped it without going and +digging through CI logs or git. Both `mark_stories_deployed.py` and +`reconcile_deploy_ready.py` now post a short, factual comment +(`POST /stories/{id}/comments`) immediately after a story is ACTUALLY +transitioned by that run — never for already-Done/skipped stories, and +never merely because a dry-run run classified something as a candidate. + +The POST mechanics (`build/ci/shortcut_comment.py`) are shared between the +two scripts for the same drift-prevention reason `shortcut_pr_guards.py` +is shared for the PR-level guards — one implementation, not two copies that +could quietly diverge. The comment TEXT is deliberately **not** shared, +because the two scripts know different things about which release actually +shipped a story: + +- **`mark_stories_deployed.py`** is reading THIS release's own + shipped-stories.json, so it already has `version`, `chart_version` and + `release_date` for exactly the release a story just shipped in — the + comment states that directly, plus the PR(s) that carried it (resolved + from the JSON's own `commits` list). +- **`reconcile_deploy_ready.py`** does NOT know that — a story it backfills + shipped in some EARLIER release, and if its comment named the CURRENT + prod tag, a reader would reasonably conclude that story shipped in + TODAY's release. That is exactly the "old features shipped today" error + class the `shipped-stories.json` / reconcile-report separation already + documented above exists to prevent — just showing up in a Shortcut + comment instead of a Slack post. So it instead asks git for the TRUE + release: + + ``` + git tag --list 'prod/*' --contains --sort=creatordate | head -1 + ``` + + the first (earliest-created) `prod/*` tag that actually contains the + winning PR's merge commit — the release that really carried it. Note the + ascending `--sort=creatordate` here, the OPPOSITE of + `resolve_default_prod_tag`'s `-creatordate`: that one wants the newest + tag (today's release); this one wants the OLDEST tag that still contains + the commit, i.e. the first release it ever reached. If that lookup can't + be resolved for any reason (shallow checkout, a genuine gap in tag + history, ...), the comment degrades HONESTLY — it says only that the + story was detected as already present in production as of the current + prod tag, and names the PR. It never guesses or implies a specific + release. + +Safety properties, both scripts: + +- Posted ONLY after a transition actually succeeds; a failed transition + posts nothing. +- Never posted in `--dry-run` (or reconcile's default no-`--apply` mode) — + the report instead shows what WOULD be posted (`would_comment` in the + JSON, a preview line in the stdout summary). +- A comment failure is logged (`WARNING` to stderr) and recorded + (`comment_failed` in the JSON summary/report) but never fails the run or + rolls back the already-successful transition — the state change is the + valuable, already-durable part; the comment is a best-effort annotation + on top of it. +- `--no-comment` on both scripts opts out of the annotation entirely while + still transitioning. +- No separate dedupe index: a transitioned story leaves Deploy Ready, so a + re-run's search/classify simply never sees it again — idempotency falls + out of the state machine for free. + ## What's already wired up in this repo - `helm-chart/sefaria/templates/analysistemplate/rollout-complete.yaml` — diff --git a/build/ci/mark_stories_deployed.py b/build/ci/mark_stories_deployed.py index 97e4ab09bd..57c73d1add 100755 --- a/build/ci/mark_stories_deployed.py +++ b/build/ci/mark_stories_deployed.py @@ -36,6 +36,22 @@ silently lose stories that shipped but that shipped_stories.py could not look up. +Immediately after a story is ACTUALLY transitioned by this run (never for +already_done/skipped stories, and never merely because it was a candidate), +a short write-back comment is posted on it via +`POST /stories/{id}/comments` (shortcut_comment.py, shared with +reconcile_deploy_ready.py's own write-back) naming the release that shipped +it: this run's own --input JSON already carries `version`, `chart_version` +and `release_date` for exactly that release, so there's no ambiguity to +guard against here the way reconcile_deploy_ready.py's backfill sweep has +to (see that script's own docstring). The PR(s) that carried the story, if +resolvable from --input's `commits` list, are included as GitHub links. +A comment failure is logged and recorded in `comment_failed` but never +fails the run or rolls back the transition -- the state change is the +valuable, already-durable part; the comment is a best-effort annotation on +top of it. Posting is skipped entirely in --dry-run (which instead prints +what WOULD be posted) and with --no-comment. + Whenever the input's "stories" list is non-empty and at least one story was skipped for a reason other than already being Done, this script prints a WARNING to stderr naming those stories -- even in a MIXED release where some @@ -62,6 +78,14 @@ import urllib.error import urllib.request +# The story-comment POST mechanics are shared with reconcile_deploy_ready.py +# -- see shortcut_comment.py's own docstring for why. build/ci is not a +# package (see tests/conftest.py), but a plain sibling-module import works +# both when this file is run directly (python3 puts its own directory on +# sys.path[0]) and under pytest (the test conftest adds build/ci to +# sys.path the same way). +import shortcut_comment + SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" # Sefaria's "Standard" Shortcut workflow: "Deploy Ready" -> "Done". These are @@ -74,6 +98,10 @@ DEFAULT_FROM_STATE_ID = 500000045 DEFAULT_DONE_STATE_ID = 500000010 +# Used only to build a GitHub PR link in the write-back comment -- this +# script never calls `gh` or the GitHub API itself. +DEFAULT_REPO = "Sefaria/Sefaria-Project" + def die(message: str) -> None: print(f"ERROR: {message}", file=sys.stderr) @@ -151,6 +179,57 @@ def transition_stories(to_transition, done_state_id, token, max_workers=8): return transitioned, failed +def _pr_numbers_for_story(story_id, commits): + """PR numbers of every commit in this release's own `commits` list + (from --input's shipped-stories.json) that carried this story id, in + the order they appear there. A story can ship via more than one + commit/PR in the same release (e.g. a same-day fix-up), so this + collects all of them rather than just the first match. Returns [] + (not an error) when none are resolvable -- an older shipped-stories.json + without a `commits` key, or a story whose only carrying commit had no + PR number, both degrade to "no PR reference available" in the comment + text rather than raising.""" + sid = str(story_id) + numbers = [] + seen = set() + for c in commits: + pr_number = c.get("pr_number") + if pr_number and sid in (c.get("story_ids") or []) and pr_number not in seen: + seen.add(pr_number) + numbers.append(pr_number) + return numbers + + +def _release_comment_text(story, data, repo): + """Write-back comment text for a story THIS RUN actually transitioned. + Unlike reconcile_deploy_ready.py's backfill sweep, there's no release + ambiguity to resolve here: --input IS this release's own + shipped-stories.json, so `version`/`chart_version`/`release_date` are + exactly the release this story just shipped in -- state that fact + directly rather than re-deriving it from git the way the sweep has to + for stories it didn't just observe shipping in real time.""" + version = data.get("version") or "an unresolved version" + chart_version = data.get("chart_version") + release_date = data.get("release_date") + # Human-readable date only (no time-of-day/timezone noise) -- release_date + # is an ISO 8601 timestamp like "2026-08-31T07:17:36Z". + release_date_human = release_date.split("T")[0] if release_date else "an unresolved date" + + pr_numbers = _pr_numbers_for_story(story.get("id"), data.get("commits") or []) + if pr_numbers: + pr_line = "PR(s): " + ", ".join(f"https://github.com/{repo}/pull/{n}" for n in pr_numbers) + else: + pr_line = "PR(s): not available in this release's shipped-stories data." + + chart_part = f" (chart {chart_version})" if chart_version else "" + + return ( + "\U0001F916 Automated update — posted by the prod release pipeline; no reply expected.\n" + f"Shipped in prod release {version}{chart_part}, released {release_date_human}.\n" + f"{pr_line}" + ) + + def _ids(stories): return sorted((s.get("id") for s in stories), key=lambda x: (x is None, x)) @@ -186,6 +265,11 @@ def build_arg_parser(): parser.add_argument("--workflow-id", type=int, default=DEFAULT_WORKFLOW_ID) parser.add_argument("--from-state-id", type=int, default=DEFAULT_FROM_STATE_ID) parser.add_argument("--done-state-id", type=int, default=DEFAULT_DONE_STATE_ID) + parser.add_argument("--repo", default=DEFAULT_REPO, help=f"GitHub repo for the comment's PR link(s), default {DEFAULT_REPO}") + parser.add_argument( + "--no-comment", action="store_true", + help="Transition stories but skip posting the write-back release comment.", + ) parser.add_argument("--max-workers", type=int, default=8) return parser @@ -218,6 +302,9 @@ def main(): stories, args.workflow_id, args.from_state_id, args.done_state_id, ) + comment_posted = [] + comment_failed = [] + if args.dry_run: for story in to_transition: print( @@ -226,11 +313,38 @@ def main(): f"{args.from_state_id} to {args.done_state_id}", file=sys.stderr, ) + # Preview-only: never calls Shortcut. Shown so a --dry-run run + # says what it WOULD post, matching how it already says what it + # would transition. + if not args.no_comment: + preview = "\n".join(f"DRY RUN: {line}" for line in _release_comment_text(story, data, args.repo).splitlines()) + print(f"DRY RUN: would post comment on story {story.get('id')}:\n{preview}", file=sys.stderr) transitioned = _ids(to_transition) failed = [] else: transitioned, failed = transition_stories(to_transition, args.done_state_id, token, args.max_workers) + # Comment ONLY on a story THIS RUN actually transitioned -- never + # already_done/skipped (those paths never reach to_transition at + # all), and never a story that was a to_transition CANDIDATE but + # whose PUT itself failed (excluded via transitioned_ids below). + # transitions are naturally once-only (a transitioned story leaves + # Deploy Ready, so classify_stories routes it to already_done on any + # re-run) -- no separate dedupe index is needed to keep a re-run + # from re-posting. + if not args.no_comment: + transitioned_ids = set(transitioned) + for story in to_transition: + if story.get("id") not in transitioned_ids: + continue + comment_text = _release_comment_text(story, data, args.repo) + sid, ok, err = shortcut_comment.post_story_comment(story.get("id"), comment_text, token) + if ok: + comment_posted.append(sid) + else: + warn(f"Failed to post release comment on story {sid}: {err}") + comment_failed.append({"id": sid, "error": err}) + skipped_detail = _skipped_detail(skipped_other_state, skipped_different_workflow) summary = { @@ -241,6 +355,8 @@ def main(): "skipped_other_state": len(skipped_other_state), "skipped_different_workflow": len(skipped_different_workflow), "failed": len(failed), + "comment_posted": len(comment_posted), + "comment_failed": len(comment_failed), }, "transitioned": sorted(transitioned, key=lambda x: (x is None, x)), "already_done": _ids(already_done), @@ -248,6 +364,8 @@ def main(): "skipped_different_workflow": _ids(skipped_different_workflow), "skipped_detail": skipped_detail, "failed": sorted(failed, key=lambda x: (x is None, x)), + "comment_posted": sorted(comment_posted, key=lambda x: (x is None, x)), + "comment_failed": sorted(comment_failed, key=lambda d: (d["id"] is None, d["id"])), "hydrated": hydrated, "unresolved_story_ids": sorted(unresolved_story_ids), } diff --git a/build/ci/reconcile_deploy_ready.py b/build/ci/reconcile_deploy_ready.py index a91745c5b0..7884767815 100644 --- a/build/ci/reconcile_deploy_ready.py +++ b/build/ci/reconcile_deploy_ready.py @@ -90,6 +90,33 @@ shipped today. Reconciliation transitions Shortcut state only; it has no opinion about what today's release notes should say. +Immediately after a story is ACTUALLY transitioned by this run, a short +write-back comment is posted on it via `POST /stories/{id}/comments` +(shortcut_comment.py, shared with mark_stories_deployed.py's own +write-back) -- but the SAME "old features shipped today" mistake this +script's whole shipped-stories.json separation exists to avoid can just as +easily happen inside a single Shortcut comment, so the comment text is +built differently here than in mark_stories_deployed.py. That script knows +the current release's own version/chart/date directly (it's reading that +release's own shipped-stories.json); this script does NOT -- a story it +backfills shipped in some EARLIER release, and naming the CURRENT prod tag +in its comment would tell a reader it shipped TODAY, which is false. So +this script instead asks git for the TRUE release: `git tag --list +'prod/*' --contains --sort=creatordate | head -1` -- the +first (earliest-created) prod/* tag that actually contains the winning +PR's merge commit, i.e. the release that really carried it. If that can't +be resolved for any reason (shallow checkout, tag history gap, ...), the +comment degrades HONESTLY: it says only that the story was detected as +already present in production as of the current prod tag, and names the +PR -- it never guesses or implies a specific release. + +A comment is posted ONLY for a story this run actually transitioned -- +never for pending/triage, and never merely because a dry-run run +classified it as shipped. A comment failure is logged and recorded in +`comment_failed` but never fails the run or rolls back the transition. +Posting is skipped entirely in dry-run (which instead reports what WOULD +be posted) and with --no-comment. + Usage: python3 reconcile_deploy_ready.py [--dry-run] python3 reconcile_deploy_ready.py --apply @@ -127,6 +154,13 @@ # conftest adds build/ci to sys.path the same way). import shortcut_pr_guards +# The story-comment POST mechanics are shared with mark_stories_deployed.py +# -- see shortcut_comment.py's own docstring for why (and for why the +# comment TEXT itself is deliberately NOT shared -- this script and +# mark_stories_deployed.py know different things about which release +# actually shipped a story). +import shortcut_comment + SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" SHORTCUT_API_ROOT = "https://api.app.shortcut.com" @@ -279,6 +313,75 @@ def is_ancestor_of_prod(oid, prod_tag): return None +def resolve_shipping_release_tag(oid): + """The TRUE release that shipped commit `oid`: the first (earliest + created) `prod/*` tag that actually contains it. Used ONLY for the + write-back comment's text -- see the module docstring for why this + script cannot just name the current `--prod-tag` the way + mark_stories_deployed.py names its own release: a story backfilled by + this sweep almost never shipped in the CURRENT release, and a comment + claiming otherwise would be actively misleading, not just imprecise. + + `--sort=creatordate` (ascending, oldest first) is deliberate and the + OPPOSITE of resolve_default_prod_tag's `-creatordate` above -- that one + wants the newest tag (today's release); this one wants the OLDEST tag + that still contains the commit, i.e. the first release it ever + reached. Returns None if the lookup fails outright (non-fatal -- + logged and the caller degrades the comment text honestly) or if no + prod/* tag contains it at all (e.g. a shallow checkout, or a genuine + gap in tag history) -- both cases must never be guessed past.""" + proc = subprocess.run( + ["git", "tag", "--list", "prod/*", "--contains", oid, "--sort=creatordate"], + capture_output=True, + text=True, + ) + if proc.returncode != 0: + warn(f"git tag --list --contains {oid} failed: {proc.stderr.strip()}") + return None + tags = [t for t in proc.stdout.splitlines() if t.strip()] + return tags[0] if tags else None + + +def _reconcile_comment_text(story_entry, oid_by_pr, prod_tag, repo): + """Write-back comment text for a story THIS SWEEP actually transitioned. + See resolve_shipping_release_tag and the module docstring for why this + resolves the TRUE shipping release from git rather than naming the + current --prod-tag: naming the current release for a backfilled story + would be exactly the "old features shipped today" mistake this whole + script's two-output separation (never touching shipped-stories.json) + exists to prevent -- just showing up in a Shortcut comment instead of a + Slack post.""" + pr_numbers = story_entry.get("shipped_via_prs") or [] + if pr_numbers: + pr_line = "PR(s): " + ", ".join(f"https://github.com/{repo}/pull/{n}" for n in pr_numbers) + else: + pr_line = "PR(s): unavailable." + + shipping_tag = None + if pr_numbers: + oid = oid_by_pr.get(pr_numbers[0]) + if oid: + shipping_tag = resolve_shipping_release_tag(oid) + + if shipping_tag: + headline = ( + f"Detected as shipped in {shipping_tag} — found while reconciling the Deploy " + "Ready backlog (not part of today's release)." + ) + else: + headline = ( + f"Detected as already present in production as of {prod_tag} — found while " + "reconciling the Deploy Ready backlog; the exact shipping release could not be " + "determined." + ) + + return ( + "\U0001F916 Automated update — posted by the Deploy Ready reconciliation sweep; no reply expected.\n" + f"{headline}\n" + f"{pr_line}" + ) + + def transition_story(story_id, done_state_id, token): """PUT a workflow_state_id update to Shortcut. Mirrors mark_stories_deployed.transition_story exactly (kept as its own copy @@ -411,6 +514,13 @@ def print_summary(report): else: status = f"FAILED: {s.get('transition_error')}" print(f" {s['id']} {s.get('name', '')!r} via PR(s) {s.get('shipped_via_prs')} [{status}]") + if s.get("would_comment"): + preview = "\n".join(f" {line}" for line in s["would_comment"].splitlines()) + print(f" would post comment:\n{preview}") + elif s.get("comment_posted"): + print(" comment posted") + elif s.get("comment_error"): + print(f" comment FAILED: {s['comment_error']}") print(f"--- pending ({len(report['pending'])}) ---") for s in report["pending"]: @@ -456,6 +566,10 @@ def build_arg_parser(): f"default {DEFAULT_TARGET_BRANCH!r}", ) parser.add_argument("--out", help="Write the machine-readable JSON report to this path") + parser.add_argument( + "--no-comment", action="store_true", + help="Transition stories but skip posting the write-back release comment.", + ) parser.add_argument("--max-workers", type=int, default=8) return parser @@ -493,6 +607,8 @@ def main(): apply_mutations = args.apply and not args.dry_run failed_transitions = [] + comment_posted = [] + comment_failed = [] if apply_mutations: transitioned_ids, failed_transitions = transition_stories( [s["id"] for s in shipped], DONE_STATE_ID, token, args.max_workers @@ -503,9 +619,33 @@ def main(): s["transitioned"] = s["id"] in transitioned_set if s["id"] in failed_by_id: s["transition_error"] = failed_by_id[s["id"]] + + # Comment ONLY on a story THIS RUN actually transitioned -- never a + # candidate whose PUT itself failed, and never pending/triage. + # Re-runs never re-post: a transitioned story leaves Deploy Ready, + # so it's simply absent from the NEXT run's search results -- no + # separate dedupe index is needed. + if not args.no_comment: + for s in shipped: + if not s["transitioned"]: + continue + comment_text = _reconcile_comment_text(s, oid_by_pr, prod_tag, args.repo) + sid, ok, err = shortcut_comment.post_story_comment(s["id"], comment_text, token) + if ok: + comment_posted.append(sid) + s["comment_posted"] = True + else: + warn(f"Failed to post release comment on story {sid}: {err}") + comment_failed.append({"id": sid, "error": err}) + s["comment_error"] = err else: for s in shipped: s["transitioned"] = None # not attempted -- dry-run + # Preview-only: never calls Shortcut, but DOES shell out to git + # (resolve_shipping_release_tag) -- read-only local introspection, + # not a live mutation, so it's safe to compute even in dry-run. + if not args.no_comment: + s["would_comment"] = _reconcile_comment_text(s, oid_by_pr, prod_tag, args.repo) report = { "prod_tag": prod_tag, @@ -515,10 +655,14 @@ def main(): "shipped": len(shipped), "pending": len(pending), "triage": len(triage), + "comment_posted": len(comment_posted), + "comment_failed": len(comment_failed), }, "shipped": shipped, "pending": pending, "triage": triage, + "comment_posted": sorted(comment_posted), + "comment_failed": sorted(comment_failed, key=lambda d: (d["id"] is None, d["id"])), } print_summary(report) diff --git a/build/ci/shortcut_comment.py b/build/ci/shortcut_comment.py new file mode 100644 index 0000000000..87663b80b0 --- /dev/null +++ b/build/ci/shortcut_comment.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +""" +Shared "post a comment on a Shortcut story" helper. + +Today this pipeline only ever WRITES a story's workflow state (a PUT that +changes `workflow_state_id`) -- nothing records WHICH release actually +carried a story, so a person reading the story inside Shortcut can see it +became Done but has no way to tell what shipped it without going and +digging through CI logs or git. Both `mark_stories_deployed.py` (this +release's own shipped stories) and `reconcile_deploy_ready.py` (older +stories backfilled by the org-wide sweep) close that gap by posting a +short, factual write-back comment immediately after a successful +transition -- but they need the exact same POST mechanics and the exact +same non-fatal error handling around it, so that lives here once. Two +copies of this would be exactly the drift failure the shared +shortcut_pr_guards.py module already exists to prevent for the PR-level +guards -- same reasoning, same fix. + +What differs between the two callers is not HOW to post a comment, but +WHAT the comment says: `mark_stories_deployed.py` knows the current +release's own version/chart/date (it's reading that release's own +shipped-stories.json) and can just say so. `reconcile_deploy_ready.py` +does NOT know that -- a story it backfills shipped in some EARLIER +release, and if its comment named the CURRENT prod tag, a reader would +reasonably conclude that story shipped in TODAY's release. That's exactly +the "old features shipped today" error class the shipped-stories.json / +reconcile-report separation in this codebase's other docstrings exists to +avoid, just showing up in a different place (a Shortcut comment instead of +a Slack post). So comment-TEXT construction stays in each caller, where +the release-identity knowledge already lives; only the POST mechanics are +shared here. + +Stdlib only -- no third-party dependencies, matching both callers' +dependency posture. +""" + +import json +import urllib.error +import urllib.request + +SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" + + +def post_story_comment(story_id, text, token): + """POST a comment onto a story (`POST /stories/{id}/comments`). + Returns (story_id, ok, error). Mirrors the transition_story functions + in both callers: a failure here must never raise, and must never be + conflated with a failed transition -- by the time this is ever called, + the state change has ALREADY succeeded. The comment is a best-effort + annotation on top of a real, already-durable state change, not a + precondition for it -- so a comment failure is reported (the caller + warns and tracks it) but never rolls anything back and never fails the + run on its own.""" + url = f"{SHORTCUT_API_BASE}/stories/{story_id}/comments" + body = json.dumps({"text": text}).encode("utf-8") + req = urllib.request.Request(url, data=body, method="POST") + req.add_header("Shortcut-Token", token) + req.add_header("Content-Type", "application/json") + try: + with urllib.request.urlopen(req, timeout=20) as resp: + resp.read() + return story_id, True, None + except urllib.error.HTTPError as e: + return story_id, False, f"HTTP {e.code} {e.reason}" + except Exception as e: # noqa: BLE001 - a comment failure must never abort the run or roll back the transition + return story_id, False, str(e) diff --git a/build/ci/tests/test_mark_stories_deployed.py b/build/ci/tests/test_mark_stories_deployed.py index a1fd43b1ab..883af03fb0 100644 --- a/build/ci/tests/test_mark_stories_deployed.py +++ b/build/ci/tests/test_mark_stories_deployed.py @@ -135,6 +135,8 @@ def _boom(*args, **kwargs): "skipped_other_state": 1, "skipped_different_workflow": 1, "failed": 0, + "comment_posted": 0, + "comment_failed": 0, } @@ -170,6 +172,9 @@ def _boom(*args, **kwargs): # --- Live run (mocked urllib): transitions only the Deploy Ready bucket - def test_live_run_transitions_only_deploy_ready_stories(monkeypatch, tmp_path): + """--no-comment here to keep this test scoped to transition mechanics + only -- the write-back comment behavior has its own dedicated tests + below.""" monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") calls = [] @@ -196,7 +201,7 @@ def _fake_urlopen(req, timeout=None): monkeypatch.setattr( "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), + ["mark_stories_deployed.py", "--input", str(input_path), "--no-comment", "--workflow-id", str(WORKFLOW_ID), "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], ) @@ -235,7 +240,7 @@ def read(self): monkeypatch.setattr( "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), + ["mark_stories_deployed.py", "--input", str(input_path), "--no-comment", "--workflow-id", str(WORKFLOW_ID), "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], ) @@ -473,3 +478,210 @@ def _boom(*args, **kwargs): err = capsys.readouterr().err assert "silent no-op" not in err + + +# --- write-back release comment on a successful transition -------------- +# (POST /stories/{id}/comments -- shared shortcut_comment.post_story_comment) + +RELEASE_INPUT_WITH_COMMITS = { + "version": "6.111.0-prod.2", + "chart_version": "0.87.5-prod.1", + "release_date": "2026-08-31T07:17:36Z", + "commits": [ + {"subject": "fix: a change (#3644)", "pr_number": "3644", "story_ids": ["11111"]}, + ], + "stories": [STORY_DEPLOY_READY_1], +} + + +class _OKResponse: + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def read(self): + return b"{}" + + +def test_comment_posted_on_successful_transition(monkeypatch, tmp_path): + """A story this run actually transitioned gets a write-back comment + naming THIS release's own version/chart/date and the PR that carried + it -- available directly from --input, no ambiguity to resolve (unlike + reconcile_deploy_ready.py's backfill sweep).""" + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + + calls = [] + + def _fake_urlopen(req, timeout=None): + calls.append((req.get_method(), req.full_url, req.data)) + return _OKResponse() + + monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) + + input_path = tmp_path / "shipped-stories.json" + input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") + + monkeypatch.setattr( + "sys.argv", + ["mark_stories_deployed.py", "--input", str(input_path), + "--workflow-id", str(WORKFLOW_ID), + "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], + ) + msd.main() + + put_calls = [c for c in calls if c[0] == "PUT"] + post_calls = [c for c in calls if c[0] == "POST"] + assert put_calls == [("PUT", f"{msd.SHORTCUT_API_BASE}/stories/11111", put_calls[0][2])] + assert len(post_calls) == 1 + _, url, body = post_calls[0] + assert url == f"{msd.SHORTCUT_API_BASE}/stories/11111/comments" + text = json.loads(body.decode("utf-8"))["text"] + assert "6.111.0-prod.2" in text + assert "0.87.5-prod.1" in text + assert "2026-08-31" in text + assert "https://github.com/Sefaria/Sefaria-Project/pull/3644" in text + + +def test_comment_not_posted_on_failed_transition(monkeypatch, tmp_path): + """A story whose PUT itself failed must get NO comment -- the write-back + only follows an ACTUAL transition, never a mere candidate.""" + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + + import urllib.error + + def _fake_urlopen(req, timeout=None): + if req.get_method() == "PUT": + raise urllib.error.HTTPError(req.full_url, 500, "Internal Server Error", None, None) + raise AssertionError("no comment POST must be attempted for a story whose transition failed") + + monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) + + input_path = tmp_path / "shipped-stories.json" + input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") + + monkeypatch.setattr( + "sys.argv", + ["mark_stories_deployed.py", "--input", str(input_path), + "--workflow-id", str(WORKFLOW_ID), + "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], + ) + msd.main() # must not raise despite the AssertionError path being unreachable + + +def test_comment_not_posted_in_dry_run(monkeypatch, tmp_path, capsys): + """--dry-run must post nothing -- but the report says what it WOULD + post, including the release identity and PR link.""" + monkeypatch.delenv("SHORTCUT_API_TOKEN", raising=False) + + def _boom(*args, **kwargs): + raise AssertionError("urlopen (transition or comment) must never be called in --dry-run") + + monkeypatch.setattr(msd.urllib.request, "urlopen", _boom) + + input_path = tmp_path / "shipped-stories.json" + input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") + + monkeypatch.setattr( + "sys.argv", + ["mark_stories_deployed.py", "--input", str(input_path), "--dry-run", + "--workflow-id", str(WORKFLOW_ID), + "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], + ) + msd.main() + + out = json.loads(capsys.readouterr().out) + assert out["counts"]["comment_posted"] == 0 + assert out["counts"]["comment_failed"] == 0 + + +def test_dry_run_previews_the_comment_it_would_post(monkeypatch, tmp_path, capsys): + monkeypatch.delenv("SHORTCUT_API_TOKEN", raising=False) + monkeypatch.setattr( + msd.urllib.request, "urlopen", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("urlopen must never be called in --dry-run")), + ) + + input_path = tmp_path / "shipped-stories.json" + input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") + + monkeypatch.setattr( + "sys.argv", + ["mark_stories_deployed.py", "--input", str(input_path), "--dry-run", + "--workflow-id", str(WORKFLOW_ID), + "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], + ) + msd.main() + + err = capsys.readouterr().err + assert "would post comment on story 11111" in err + assert "6.111.0-prod.2" in err + assert "https://github.com/Sefaria/Sefaria-Project/pull/3644" in err + + +def test_comment_not_posted_with_no_comment_flag(monkeypatch, tmp_path): + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + + def _fake_urlopen(req, timeout=None): + if req.get_method() == "POST": + raise AssertionError("--no-comment must suppress the write-back comment entirely") + return _OKResponse() + + monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) + + input_path = tmp_path / "shipped-stories.json" + input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") + + monkeypatch.setattr( + "sys.argv", + ["mark_stories_deployed.py", "--input", str(input_path), "--no-comment", + "--workflow-id", str(WORKFLOW_ID), + "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], + ) + msd.main() # must not raise -- proves no POST was attempted + + +def test_comment_api_failure_does_not_fail_the_run_and_is_reported(monkeypatch, tmp_path, capsys): + """A failed comment POST must never fail the run or roll back the + transition -- it's reported (warned + counted) and nothing else.""" + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + + import urllib.error + + def _fake_urlopen(req, timeout=None): + if req.get_method() == "PUT": + return _OKResponse() + raise urllib.error.HTTPError(req.full_url, 503, "Service Unavailable", None, None) + + monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) + + input_path = tmp_path / "shipped-stories.json" + input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") + + monkeypatch.setattr( + "sys.argv", + ["mark_stories_deployed.py", "--input", str(input_path), + "--workflow-id", str(WORKFLOW_ID), + "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], + ) + # Must return normally -- a comment failure is never fatal. + msd.main() + + captured = capsys.readouterr() + out = json.loads(captured.out) + assert out["counts"]["transitioned"] == 1 + assert out["counts"]["comment_failed"] == 1 + assert out["comment_failed"] == [{"id": 11111, "error": "HTTP 503 Service Unavailable"}] + assert "Failed to post release comment on story 11111" in captured.err + + +def test_release_comment_text_falls_back_when_no_pr_reference_available(): + """A story whose carrying commit has no PR number (or whose release's + commits list doesn't mention it at all) must still get a comment -- + just without a PR link, not a crash.""" + data = {"version": "6.111.0-prod.2", "chart_version": "0.87.5-prod.1", + "release_date": "2026-08-31T07:17:36Z", "commits": []} + text = msd._release_comment_text({"id": 11111}, data, "Sefaria/Sefaria-Project") + assert "not available" in text + assert "6.111.0-prod.2" in text diff --git a/build/ci/tests/test_reconcile_deploy_ready.py b/build/ci/tests/test_reconcile_deploy_ready.py index 3eca7629ff..6016ec64c3 100644 --- a/build/ci/tests/test_reconcile_deploy_ready.py +++ b/build/ci/tests/test_reconcile_deploy_ready.py @@ -356,10 +356,17 @@ class _Proc: # --- End-to-end main(): dry-run is the default and mutates nothing ------- def _make_main_env(monkeypatch, tmp_path, stories, prod_tag="prod/1.0", oid_by_pr=None, - ancestor_result=True, argv_extra=None): + ancestor_result=True, argv_extra=None, shipping_tag=None, + mock_post_comment=True): """Wire main() end-to-end with every I/O boundary mocked: Shortcut - search, gh merge-commit lookup, and git ancestry -- mirroring - _run_main_with_commits in test_shipped_stories.py.""" + search, gh merge-commit lookup, git ancestry, the git-based + shipping-release lookup, and (by default) the comment POST itself -- + mirroring _run_main_with_commits in test_shipped_stories.py. + resolve_shipping_release_tag defaults to a plain lambda returning + `shipping_tag` (None unless overridden) rather than shelling out to + real git, so tests that don't care about that specific behavior stay + hermetic; the dedicated tests for it below monkeypatch it themselves + when they need to exercise its own git-shelling logic.""" monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") monkeypatch.setattr(rdr, "search_deploy_ready_stories", lambda token: stories) monkeypatch.setattr(rdr, "resolve_default_prod_tag", lambda: prod_tag) @@ -369,6 +376,12 @@ def _make_main_env(monkeypatch, tmp_path, stories, prod_tag="prod/1.0", oid_by_p lambda pr_numbers, repo, max_workers=8: {n: oid_map[n] for n in pr_numbers if n in oid_map}, ) monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: ancestor_result) + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: shipping_tag) + if mock_post_comment: + monkeypatch.setattr( + rdr.shortcut_comment, "post_story_comment", + lambda story_id, text, token: (story_id, True, None), + ) argv = ["reconcile_deploy_ready.py"] + (argv_extra or []) monkeypatch.setattr("sys.argv", argv) @@ -390,10 +403,13 @@ def _boom(*args, **kwargs): def test_main_apply_transitions_shipped_stories(monkeypatch, tmp_path, capsys): + """--no-comment here to keep this test scoped to transition mechanics + only -- the write-back comment behavior has its own dedicated tests + below.""" story = _story(11111, pull_requests=[_pr(3606)]) _make_main_env( monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - argv_extra=["--apply"], + argv_extra=["--apply", "--no-comment"], ) calls = [] @@ -437,7 +453,10 @@ def test_main_writes_out_json_report(monkeypatch, tmp_path): rdr.main() report = json.loads(out_path.read_text(encoding="utf-8")) - assert report["counts"] == {"total": 2, "shipped": 1, "pending": 0, "triage": 1} + assert report["counts"] == { + "total": 2, "shipped": 1, "pending": 0, "triage": 1, + "comment_posted": 0, "comment_failed": 0, + } assert report["applied"] is False assert [s["id"] for s in report["shipped"]] == [11111] assert [s["id"] for s in report["triage"]] == [22222] @@ -452,7 +471,10 @@ def test_main_pending_bucket_when_qualifying_pr_not_yet_in_prod(monkeypatch, tmp ) rdr.main() report = json.loads(out_path.read_text(encoding="utf-8")) - assert report["counts"] == {"total": 1, "shipped": 0, "pending": 1, "triage": 0} + assert report["counts"] == { + "total": 1, "shipped": 0, "pending": 1, "triage": 0, + "comment_posted": 0, "comment_failed": 0, + } assert report["pending"][0]["id"] == 11111 @@ -522,3 +544,225 @@ def test_resolve_default_prod_tag_dies_with_no_tags(monkeypatch): monkeypatch.setattr(rdr, "run_git", lambda args: "") with pytest.raises(SystemExit): rdr.resolve_default_prod_tag() + + +# --- resolve_shipping_release_tag: the TRUE (earliest) release, not the -- +# --- current --prod-tag -- see the module docstring for why this matters -- + +def test_resolve_shipping_release_tag_picks_earliest_containing_tag(monkeypatch): + """--sort=creatordate (ascending) here is the OPPOSITE of + resolve_default_prod_tag's -creatordate -- this wants the FIRST release + that ever contained the commit, not the newest tag overall.""" + class _Proc: + returncode = 0 + stdout = "prod/6.100.0-prod.1+chart.0.85.8-prod.1\nprod/6.111.0-prod.2+chart.0.87.5-prod.1\n" + stderr = "" + + captured_args = {} + + def _fake_run(args, **kwargs): + captured_args["args"] = args + return _Proc() + + monkeypatch.setattr(rdr.subprocess, "run", _fake_run) + tag = rdr.resolve_shipping_release_tag("abc123") + assert tag == "prod/6.100.0-prod.1+chart.0.85.8-prod.1" + assert "--contains" in captured_args["args"] + assert "abc123" in captured_args["args"] + assert "--sort=creatordate" in captured_args["args"] # ascending, not -creatordate + + +def test_resolve_shipping_release_tag_returns_none_when_no_tag_contains_it(monkeypatch): + class _Proc: + returncode = 0 + stdout = "" + stderr = "" + + monkeypatch.setattr(rdr.subprocess, "run", lambda *a, **k: _Proc()) + assert rdr.resolve_shipping_release_tag("abc123") is None + + +def test_resolve_shipping_release_tag_returns_none_and_warns_on_git_failure(monkeypatch, capsys): + class _Proc: + returncode = 128 + stdout = "" + stderr = "fatal: malformed object name abc123" + + monkeypatch.setattr(rdr.subprocess, "run", lambda *a, **k: _Proc()) + assert rdr.resolve_shipping_release_tag("abc123") is None + assert "WARNING" in capsys.readouterr().err + + +# --- write-back release comment: posted / not posted / dry-run preview --- +# --- / --no-comment / API failure / honest degrade (regression coverage -- +# --- for the promotion-PR-style "old features shipped today" mistake, -- +# --- just in a Shortcut comment instead of a Slack post) ----------------- + +def test_main_posts_comment_after_a_real_transition(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3606)]) + calls = [] + + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--apply"], shipping_tag="prod/6.100.0-prod.1+chart.0.85.8-prod.1", + mock_post_comment=False, + ) + monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, True, None)) + monkeypatch.setattr( + rdr.shortcut_comment, "post_story_comment", + lambda story_id, text, token: (calls.append((story_id, text)), story_id, True, None)[1:], + ) + + rdr.main() + + assert len(calls) == 1 + posted_id, text = calls[0] + assert posted_id == 11111 + assert "prod/6.100.0-prod.1+chart.0.85.8-prod.1" in text + assert "not part of today's release" in text + assert "https://github.com/Sefaria/Sefaria-Project/pull/3606" in text + # Must NOT name the CURRENT prod tag as if the story shipped in it. + assert "prod/1.0" not in text + + +def test_main_does_not_post_comment_when_transition_fails(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3606)]) + + def _boom_post(*args, **kwargs): + raise AssertionError("no comment must be posted for a story whose transition failed") + + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--apply"], mock_post_comment=False, + ) + monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, False, "HTTP 500 Internal Server Error")) + monkeypatch.setattr(rdr.shortcut_comment, "post_story_comment", _boom_post) + + with pytest.raises(SystemExit): + rdr.main() # transition failure still exits non-zero + + +def test_main_does_not_post_comment_in_dry_run(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3606)]) + + def _boom_post(*args, **kwargs): + raise AssertionError("no comment must be posted in dry-run") + + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + mock_post_comment=False, # dry-run (no --apply): must never even try to POST + ) + monkeypatch.setattr(rdr.shortcut_comment, "post_story_comment", _boom_post) + rdr.main() # must not raise -- proves no POST was attempted + + +def test_main_dry_run_report_previews_the_comment_it_would_post(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3606)]) + out_path = tmp_path / "report.json" + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + shipping_tag="prod/6.100.0-prod.1+chart.0.85.8-prod.1", + argv_extra=["--out", str(out_path)], + ) + rdr.main() + + report = json.loads(out_path.read_text(encoding="utf-8")) + assert report["counts"]["comment_posted"] == 0 + would_comment = report["shipped"][0]["would_comment"] + assert "prod/6.100.0-prod.1+chart.0.85.8-prod.1" in would_comment + + +def test_main_no_comment_flag_suppresses_posting(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3606)]) + + def _boom_post(*args, **kwargs): + raise AssertionError("--no-comment must suppress the write-back comment entirely") + + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--apply", "--no-comment"], mock_post_comment=False, + ) + monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, True, None)) + monkeypatch.setattr(rdr.shortcut_comment, "post_story_comment", _boom_post) + + rdr.main() # must not raise -- proves no POST was attempted + + +def test_main_no_comment_flag_suppresses_dry_run_preview_too(monkeypatch, tmp_path): + story = _story(11111, pull_requests=[_pr(3606)]) + out_path = tmp_path / "report.json" + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--no-comment", "--out", str(out_path)], + ) + rdr.main() + report = json.loads(out_path.read_text(encoding="utf-8")) + assert "would_comment" not in report["shipped"][0] + + +def test_main_comment_api_failure_does_not_fail_the_run_and_is_reported(monkeypatch, tmp_path, capsys): + """A failed comment POST must never fail the run or roll back the + transition -- it's reported (warned + counted) and nothing else.""" + story = _story(11111, pull_requests=[_pr(3606)]) + out_path = tmp_path / "report.json" + + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + argv_extra=["--apply", "--out", str(out_path)], mock_post_comment=False, + ) + monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, True, None)) + monkeypatch.setattr( + rdr.shortcut_comment, "post_story_comment", + lambda story_id, text, token: (story_id, False, "HTTP 503 Service Unavailable"), + ) + + rdr.main() # must return normally -- a comment failure is never fatal + + report = json.loads(out_path.read_text(encoding="utf-8")) + assert report["counts"]["shipped"] == 1 + assert report["counts"]["comment_failed"] == 1 + assert report["comment_failed"] == [{"id": 11111, "error": "HTTP 503 Service Unavailable"}] + assert "Failed to post release comment on story 11111" in capsys.readouterr().err + + +def test_main_comment_names_the_contains_derived_release_not_the_current_one(monkeypatch, tmp_path): + """Regression coverage for the exact mistake this feature exists to + avoid: a story backfilled by this sweep shipped in an EARLIER release, + and the comment must name THAT release (resolved via + resolve_shipping_release_tag's `--contains` lookup), never the current + --prod-tag / today's release.""" + story = _story(11111, pull_requests=[_pr(3606)]) + out_path = tmp_path / "report.json" + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + prod_tag="prod/9.9.9-prod.1+chart.9.9.9-prod.1", # "today's" release + shipping_tag="prod/6.100.0-prod.1+chart.0.85.8-prod.1", # the TRUE, earlier release + argv_extra=["--out", str(out_path)], + ) + rdr.main() + report = json.loads(out_path.read_text(encoding="utf-8")) + would_comment = report["shipped"][0]["would_comment"] + assert "prod/6.100.0-prod.1+chart.0.85.8-prod.1" in would_comment + assert "prod/9.9.9-prod.1+chart.9.9.9-prod.1" not in would_comment + + +def test_main_comment_degrades_honestly_when_release_tag_lookup_fails(monkeypatch, tmp_path): + """When resolve_shipping_release_tag can't determine the true release + (returns None -- e.g. a shallow checkout or a genuine history gap), the + comment must degrade to naming the current prod tag as "present as of" + language and the PR -- it must NEVER guess or imply a specific + release.""" + story = _story(11111, pull_requests=[_pr(3606)]) + out_path = tmp_path / "report.json" + _make_main_env( + monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, + prod_tag="prod/9.9.9-prod.1+chart.9.9.9-prod.1", + shipping_tag=None, # lookup failed / inconclusive + argv_extra=["--out", str(out_path)], + ) + rdr.main() + report = json.loads(out_path.read_text(encoding="utf-8")) + would_comment = report["shipped"][0]["would_comment"] + assert "present in production as of prod/9.9.9-prod.1+chart.9.9.9-prod.1" in would_comment + assert "could not be determined" in would_comment + assert "https://github.com/Sefaria/Sefaria-Project/pull/3606" in would_comment From 159e2b340676f4eca64a875b8f71c9768365eedf Mon Sep 17 00:00:00 2001 From: Yotam Fromm Date: Mon, 7 Sep 2026 09:50:50 +0300 Subject: [PATCH 3/6] feat(ci): add opt-in LLM explainer for the Deploy Ready triage backlog (sc-47043) reconcile_deploy_ready.py's triage bucket only reports raw diagnostics (no_qualifying_pr / non_standard_workflow_or_state), leaving a human to open every story and work out why individually. This adds an opt-in headless-Claude explainer step that proposes a labeled hypothesis and a suggested next action per triage story. Kept fully out of the deterministic Python: reconcile_deploy_ready.py gained no API client, just richer in-memory triage context (description, comments, per-linked-PR guard diagnostics -- all already available on the same Shortcut search response, no extra calls). A new sibling script, build/ci/triage_explainer.py, is a stdlib-only JSON transform that extracts ONLY the triage bucket into its own file -- shipped/pending are structurally absent, not merely excluded by prompt instruction, since a triage story's description/comments are contributor-controlled text. The actual `claude -p` call lives entirely in a new, opt-in workflow step, provisioned identically to the existing release-notes prose step (--allowedTools Read,Write,Glob,Grep only, no Bash, no network), and its output stays in $RUNNER_TEMP so the prose step can't pick it up. It never mutates Shortcut and never influences the shipped/pending buckets or any transition. Opt-in and off by default on every trigger path: a workflow_dispatch run via its own explain_triage input, the automatic repository_dispatch trigger via a repo-level Actions variable (vars.ENABLE_TRIAGE_EXPLAINER). The decision itself (triage_explainer.resolve_enabled) is tested Python, not inline bash. Degrades cleanly (continue-on-error, same posture as the existing bookkeeping steps) when disabled, when ANTHROPIC_API_KEY is absent, or when the call fails. Note: verified against this repo's actual configured secrets (gh secret list) that the correctly-spelled ANTHROPIC_API_KEY is the only secret that exists -- used that instead of a claimed ANTHOPIC_API_KEY misspelling, which does not exist here and would have silently disabled this feature permanently in production. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KJqBksQqzHYs3F54Y4tJRX --- .github/workflows/prod-release-notes.yaml | 148 ++++++++++-- build/ci/README-prod-release-notes.md | 136 ++++++++++- build/ci/reconcile_deploy_ready.py | 47 +++- build/ci/tests/test_reconcile_deploy_ready.py | 91 ++++++- build/ci/tests/test_triage_explainer.py | 226 ++++++++++++++++++ build/ci/triage_explainer.py | 167 +++++++++++++ 6 files changed, 786 insertions(+), 29 deletions(-) create mode 100644 build/ci/tests/test_triage_explainer.py create mode 100644 build/ci/triage_explainer.py diff --git a/.github/workflows/prod-release-notes.yaml b/.github/workflows/prod-release-notes.yaml index abf962e24d..dda279c360 100644 --- a/.github/workflows/prod-release-notes.yaml +++ b/.github/workflows/prod-release-notes.yaml @@ -7,30 +7,40 @@ name: "Prod Release Notes" # repository_dispatch here, carrying the deployed `version` and `chartVersion`. # # From there this workflow is a strict pipeline of deterministic scripts, with -# exactly one step handed to an LLM: +# two steps handed to an LLM (release-notes prose, and an OPT-IN triage +# explanation -- see step 4 below): # 1. build/ci/shipped_stories.py resolves the prod/* tag range for that # version (chartVersion disambiguates a chart-only rollout, where several # tags share the same app version), walks git log between the tags, and # hydrates the Shortcut stories those commits reference. # 2. build/ci/mark_stories_deployed.py moves each of those stories from -# Deploy Ready -> Done via the Shortcut API. A failure here (missing -# token, API error, nothing to move) is surfaced as a warning but never -# blocks steps 3-5 — see "Mark shipped stories as deployed" below. +# Deploy Ready -> Done via the Shortcut API, then posts a write-back +# comment on each naming this release. A failure here (missing token, +# API error, nothing to move) is surfaced as a warning but never blocks +# steps 3-6 — see "Mark shipped stories as deployed" below. # 3. build/ci/reconcile_deploy_ready.py sweeps EVERY non-archived Deploy # Ready story org-wide (not just this release's commit range) and # transitions any whose linked PR already reached prod, regardless of -# which release actually shipped it. This backfills stories a prior -# release's git-range-scoped run never revisited (RC2) — see that -# script's own docstring for the four guards it applies. Its output -# NEVER feeds shipped-stories.json or step 4/5 below: those stories -# shipped in EARLIER releases, and leaking them into today's -# announcement would have Slack claim old features shipped today. A -# failure here is warned/alerted the same way step 2's is, and never -# blocks steps 4-5. -# 4. The sefaria-release-notes skill (.claude/skills/sefaria-release-notes/) +# which release actually shipped it, then posts its own write-back +# comment naming the release that ACTUALLY shipped it (never this one). +# This backfills stories a prior release's git-range-scoped run never +# revisited (RC2) — see that script's own docstring for the four guards +# it applies. Its output NEVER feeds shipped-stories.json or steps 5-6 +# below: those stories shipped in EARLIER releases, and leaking them +# into today's announcement would have Slack claim old features shipped +# today. A failure here is warned/alerted the same way step 2's is, and +# never blocks steps 4-6. +# 4. build/ci/triage_explainer.py + a headless `claude -p` run (OPT-IN, +# off by default -- see "Explain Deploy Ready triage backlog" below) +# proposes a short, labeled HYPOTHESIS for why each story in step 3's +# triage bucket is stuck, and a suggested next action for a human. It +# sees ONLY the triage bucket (never shipped/pending, structurally — +# see triage_explainer.py's own docstring), never mutates Shortcut, +# and its output never reaches steps 5-6 either. +# 5. The sefaria-release-notes skill (.claude/skills/sefaria-release-notes/) # reads that JSON and writes prose only — it does not talk to GitHub or # Shortcut. -# 5. scripts/post_to_slack.py posts both generated files to Slack. +# 6. scripts/post_to_slack.py posts both generated files to Slack. # # Manual setup this depends on: see build/ci/README-prod-release-notes.md @@ -62,6 +72,11 @@ on: required: false type: boolean default: true + explain_triage: + description: "Opt-in: run the headless triage explainer over the reconcile sweep's triage bucket only (requires ANTHROPIC_API_KEY; never mutates Shortcut)" + required: false + type: boolean + default: false # A pod retry (the AnalysisTemplate Job has backoffLimit: 1) can re-fire the # repository_dispatch for the same version, and a manual workflow_dispatch @@ -94,6 +109,18 @@ jobs: # commit history. fetch-tags: true + # Moved up front (both LLM steps in this workflow need the `claude` + # CLI): the release-notes prose step below, AND the opt-in triage + # explainer, which now runs earlier in the pipeline than prose does. + # One install, shared by both, instead of two. + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install Claude Code CLI + run: npm install -g @anthropic-ai/claude-code + - name: Resolve version id: resolve env: @@ -228,13 +255,94 @@ jobs: --data "{\"text\": \":warning: Prod release notes for version ${VERSION:-unknown}: ${FAILURES}Release notes generation is continuing. Check the workflow run and move the affected stories manually.\"}" \ "$SLACK_WEBHOOK_URL" || true - - name: Set up Node - uses: actions/setup-node@v4 - with: - node-version: "20" + - name: Resolve triage explainer opt-in + id: triage_opt_in + env: + EVENT_NAME: ${{ github.event_name }} + EXPLAIN_TRIAGE_INPUT: ${{ inputs.explain_triage }} + # A repo-level Actions VARIABLE (Settings -> Secrets and + # variables -> Actions -> Variables), not a secret -- this is a + # plain on/off switch, nothing sensitive. Lets the automatic + # repository_dispatch trigger opt in too, since that trigger + # carries no workflow_dispatch-style inputs at all. + ENABLE_TRIAGE_EXPLAINER_VAR: ${{ vars.ENABLE_TRIAGE_EXPLAINER }} + run: | + # The actual decision rule (triage_explainer.resolve_enabled) is + # tested Python, not bash string comparisons duplicated here -- + # see build/ci/triage_explainer.py's own docstring. Off by + # default on every trigger path until a human explicitly flips + # one of the two switches. + ENABLED=$(python3 build/ci/triage_explainer.py resolve-enabled \ + --event-name "$EVENT_NAME" \ + --explain-triage-input "$EXPLAIN_TRIAGE_INPUT" \ + --enable-var "$ENABLE_TRIAGE_EXPLAINER_VAR") + echo "enabled=${ENABLED}" >> "$GITHUB_OUTPUT" - - name: Install Claude Code CLI - run: npm install -g @anthropic-ai/claude-code + - name: Explain Deploy Ready triage backlog (optional) + id: explain_triage + if: steps.triage_opt_in.outputs.enabled == 'true' + # Opt-in and best-effort: a triage explanation is a convenience for + # whoever works the backlog next, never a precondition for + # anything downstream. continue-on-error keeps a failure here from + # ever touching the release announcement -- same posture as "Mark + # shipped stories as deployed" / "Reconcile Deploy Ready backlog" + # above. + continue-on-error: true + env: + # This IS `ANTHROPIC_API_KEY` (correctly spelled) -- verified + # against this repo's actual configured secrets (`gh secret + # list`) rather than assumed, and it's the exact same secret the + # "Generate release notes" step below already uses. + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + if [[ -z "$ANTHROPIC_API_KEY" ]]; then + echo "ANTHROPIC_API_KEY is empty -- skipping the triage explainer. This is an opt-in convenience feature and degrades cleanly without a key; it never blocks the release announcement." + exit 0 + fi + + REPORT_FILE="$RUNNER_TEMP/reconcile-deploy-ready-report.json" + if [[ ! -s "$REPORT_FILE" ]]; then + echo "No reconcile report found at $REPORT_FILE (the reconcile step may have failed or produced nothing) -- skipping the triage explainer." + exit 0 + fi + + # Structural isolation, not a prompt instruction: this extracts + # ONLY the triage bucket (plus prod_tag) from the reconcile + # report into its own file -- "shipped" and "pending" are never + # even present in what the model reads, so there is nothing + # there for a prompt-injection path (a story's own + # description/comments are contributor-controlled text) to leak + # or influence. See triage_explainer.py's own docstring. + TRIAGE_ONLY_FILE="$RUNNER_TEMP/reconcile-triage-only.json" + python3 build/ci/triage_explainer.py extract --report "$REPORT_FILE" --out "$TRIAGE_ONLY_FILE" + + TRIAGE_COUNT=$(python3 -c "import json; print(json.load(open('$TRIAGE_ONLY_FILE'))['triage_count'])") + if [[ "$TRIAGE_COUNT" == "0" ]]; then + echo "No triage stories this run -- nothing for the explainer to explain." + exit 0 + fi + + ANNOTATED_FILE="$RUNNER_TEMP/reconcile-triage-annotated.json" + # --allowedTools instead of --dangerously-skip-permissions: same + # reasoning as "Generate release notes" below -- the input embeds + # contributor-controlled text (story descriptions, comments), a + # prompt-injection path. This agent never holds Bash or network + # tools, and it never touches Shortcut or git itself: it only + # reads the triage-only JSON and writes prose hypotheses to + # $RUNNER_TEMP, which the prose step further below never reads -- + # same "not the checkout, not a naming convention, an actual + # access boundary" reasoning as the reconcile report itself (see + # the workflow header and reconcile_deploy_ready.py's docstring). + claude -p "Read the JSON file at ${TRIAGE_ONLY_FILE}. It lists Shortcut stories an automated Deploy Ready reconciliation sweep could not classify as shipped or pending ('triage'), each with: name, description, comments, and (for a story with a linked PR that didn't qualify) linked_prs -- the PR number and exactly which shipping-evidence guard(s) it failed (not merged / wrong repo / wrong target branch), or (for a story on the wrong Shortcut workflow) its workflow_id/workflow_state_id. For EACH story in the 'triage' array, propose (a) a short hypothesis for why it is stuck, grounded ONLY in the fields given -- e.g. 'a linked PR targets preprod, not master -- likely a promotion PR was linked instead of the real feature PR', 'no PR is linked at all -- may have shipped via an unlinked PR, or a PR may still need to be opened', 'workflow/state ids do not match Standard -- this story may belong to a different team or process entirely', and (b) a short suggested next action for a human, e.g. 'search for the real feature PR and re-link it', 'no code artifact found -- consider closing manually', 'may belong to another repo's release train -- verify with that PR's author'. Do not invent PR numbers, dates, names, or any fact not present in the input. Do not recommend or imply any automatic action, transition, or comment -- every output is an unverified hypothesis for a human to check, never a finding. Prefix every hypothesis string with the literal text '[AI hypothesis, unverified] ' so it can never be mistaken for a verified cause. Write your ONLY output as JSON to ${ANNOTATED_FILE}: a list of objects, one per input triage story, each shaped {id, hypothesis, suggested_next_action}. Do not ask any clarifying questions -- proceed directly." \ + --allowedTools "Read,Write,Glob,Grep" + + if [[ -s "$ANNOTATED_FILE" ]]; then + echo "----- Triage explainer output (hypotheses only -- NOT verified findings) -----" + cat "$ANNOTATED_FILE" + echo "--------------------------------------------------------------------------------" + else + echo "Triage explainer produced no output file -- treating as a soft failure. Opt-in feature; this never blocks the run." + fi - name: Generate release notes (headless Claude Code) id: generate diff --git a/build/ci/README-prod-release-notes.md b/build/ci/README-prod-release-notes.md index 7359e5a1b6..64c76b1273 100644 --- a/build/ci/README-prod-release-notes.md +++ b/build/ci/README-prod-release-notes.md @@ -33,8 +33,15 @@ Argo post-promotion analysis (prod) shipped it (never this one) — see "Reconciliation sweep" and "Write-back release comment" below. Its output - NEVER reaches the two steps that - follow. + NEVER reaches the steps that follow. + -> build/ci/triage_explainer.py — OPT-IN, off by default: extracts + + headless `claude -p` ONLY the triage bucket into its own + file, then proposes a labeled + hypothesis + suggested next action + per triage story — see "Opt-in triage + explainer" below. Never mutates + Shortcut; its output NEVER reaches + the two steps that follow either. -> sefaria-release-notes skill — reads shipped-stories.json, writes prose only -> scripts/post_to_slack.py — posts both files to Slack @@ -267,6 +274,106 @@ Safety properties, both scripts: re-run's search/classify simply never sees it again — idempotency falls out of the state machine for free. +## Opt-in triage explainer + +`reconcile_deploy_ready.py`'s triage bucket is reported as +`reason=no_qualifying_pr` (or `non_standard_workflow_or_state`) plus raw +diagnostic fields — description, comment text, and, per linked PR, exactly +which shipping-evidence guard it failed (see `_triage_context` / +`_diagnose_linked_pr` in `reconcile_deploy_ready.py`). That still leaves a +human to open every triage story and work out each one individually. An +OPT-IN workflow step proposes a short, labeled hypothesis for each one +instead — but the pattern is deliberately the same one this workflow +already uses for release-notes prose, kept LLM-free everywhere it can be: + +``` +reconcile_deploy_ready.py — writes its report (unchanged; still + deterministic, still stdlib-only, + still no API client) + -> build/ci/triage_explainer.py — extracts ONLY the triage bucket + "extract" subcommand (+ prod_tag) into its own file + -> headless `claude -p` — reads THAT file, writes a + (in the workflow step only) {id, hypothesis, suggested_next_action} + list to a separate file +``` + +**Why a separate file, not a prompt instruction to "only look at +triage":** the full report's `shipped`/`pending` buckets must never be +visible to, scored by, or able to influence this explainer, and a triage +story's `description`/`comments` are CONTRIBUTOR-CONTROLLED TEXT — a +prompt-injection path. Rather than trust the model to honor "ignore the +other buckets" against adversarial input embedded in the very document +it's reading, `triage_explainer.py extract` simply never puts +shipped/pending data into the file the explainer is given at all. There is +nothing there to leak or be steered by, structurally, not merely by +convention — see that script's own docstring, and +`build/ci/tests/test_triage_explainer.py`, which asserts the extracted +document never contains shipped/pending data even when the source report +does. + +**Everything else about this step mirrors "Generate release notes" +below**, on purpose: + +- `--allowedTools "Read,Write,Glob,Grep"`, never + `--dangerously-skip-permissions` — no Bash, no network. This agent + cannot touch Shortcut, git, or the GitHub API even if it wanted to; + writing English from a file it's handed is the entire extent of what it + can do. +- Its output (`$RUNNER_TEMP/reconcile-triage-annotated.json`) stays in + `$RUNNER_TEMP`, never the checkout — same reasoning as the reconcile + report itself (see "Reconciliation sweep" above): the release-notes + prose step holds `Glob`+`Read` over its whole working directory, and a + distinct filename is a naming convention, not an access boundary. +- Every hypothesis string is required (by the prompt) to start with the + literal `[AI hypothesis, unverified]` marker, so it can never be + mistaken for a verified finding by whoever reads it. +- **Proposes, never decides**: no Shortcut mutation, no comment posted + from its output, no transition — of ANY kind, on ANY story. Authority + stays entirely with the deterministic layer (the four guards, the + ancestry check) and the human reading the report. An ambiguous `pr:` + lookup returning more than one story is still a deterministic + warn-and-skip in `shipped_stories.py`/`reconcile_deploy_ready.py` + (unchanged) — that's precisely the "model decides two things are + related and closes the wrong one" failure this design rejects, and the + explainer never gets a vote on it either. + +**Opt-in, off by default on every trigger path.** A `workflow_dispatch` +run opts in per-run via its own `explain_triage` input; the automatic +`repository_dispatch` trigger carries no such input at all (it's not a +`workflow_dispatch`), so it instead opts in via a repo-level Actions +*variable* (`vars.ENABLE_TRIAGE_EXPLAINER`, Settings → Secrets and +variables → Actions → **Variables**, not Secrets — it's a plain on/off +switch, nothing sensitive). The actual decision rule +(`triage_explainer.resolve_enabled`) is tested Python, not a bash string +comparison duplicated inline in the workflow — see +`build/ci/tests/test_triage_explainer.py`. It degrades cleanly and never +blocks the release announcement on any of these paths: + +- **Disabled** (the default): the step's `if:` condition is false; it + never runs at all. +- **No `ANTHROPIC_API_KEY`**: the step's own guard exits 0 immediately — + same posture as "Generate release notes" below, which requires the key + (this step is opt-in, so it degrades instead of failing the job). +- **No triage stories this run**: skipped with a short message — nothing + to explain. +- **The `claude -p` call itself fails**: `continue-on-error: true` (same + as "Mark shipped stories as deployed" / "Reconcile Deploy Ready + backlog") — a failure here is never allowed to fail the job or skip + release-notes generation and Slack posting. + +**A note on the API key secret name:** an earlier version of this +instruction claimed the repo's secret is misspelled `ANTHOPIC_API_KEY` +(no R) and that the workflow maps it deliberately. That claim was checked +against this repo's actual configured secrets (`gh secret list`) before +writing any code — the only secret that exists is the correctly-spelled +`ANTHROPIC_API_KEY`, already used by both `manual-promotion.yaml` and this +workflow's own "Generate release notes" step. This step uses that same, +correctly-spelled secret; wiring in the claimed misspelling would have +referenced a secret that doesn't exist, silently and permanently +disabling this feature in production (`secrets.ANTHOPIC_API_KEY` always +resolves to an empty string, which the step's own missing-key guard would +treat as "no key" on every single run). + ## What's already wired up in this repo - `helm-chart/sefaria/templates/analysistemplate/rollout-complete.yaml` — @@ -331,6 +438,14 @@ Add these under repo Settings → Secrets and variables → Actions: Already exist and are reused as-is: `SLACK_DEPLOY_WEBHOOK`, `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`. +Optional, only if you want the triage explainer to opt in automatically on +the real `repository_dispatch` trigger (a manual `workflow_dispatch` run +can already opt in per-run via its own `explain_triage` input without +this): add a repo-level Actions **Variable** (Settings → Secrets and +variables → Actions → **Variables** tab, NOT Secrets) named +`ENABLE_TRIAGE_EXPLAINER` set to `true`. See "Opt-in triage explainer" +above. + ## Worth verifying, not something this session could check `SLACK_URL` in `local-settings-secrets` — confirm it's actually populated @@ -379,13 +494,20 @@ anywhere — it would just be quiet. ## Running the tests The tests for these scripts (`build/ci/tests/test_shipped_stories.py`, -`test_mark_stories_deployed.py`, `test_reconcile_deploy_ready.py`) are -**not** collected by the repo's root `pytest.ini` (that config is scoped to -the Django app's own test suites), so run them by explicit path from the -repo root: +`test_mark_stories_deployed.py`, `test_reconcile_deploy_ready.py`, +`test_triage_explainer.py`) are **not** collected by the repo's root +`pytest.ini` (that config is scoped to the Django app's own test suites), +so run them by explicit path from the repo root — either the whole +directory: + +``` +python3 -m pytest build/ci/tests/ -q -p no:django -c /dev/null +``` + +or each file explicitly: ``` -python3 -m pytest build/ci/tests/test_shipped_stories.py build/ci/tests/test_mark_stories_deployed.py build/ci/tests/test_reconcile_deploy_ready.py -q -p no:django -c /dev/null +python3 -m pytest build/ci/tests/test_shipped_stories.py build/ci/tests/test_mark_stories_deployed.py build/ci/tests/test_reconcile_deploy_ready.py build/ci/tests/test_triage_explainer.py -q -p no:django -c /dev/null ``` Both flags are needed even though only explicit file paths are passed: diff --git a/build/ci/reconcile_deploy_ready.py b/build/ci/reconcile_deploy_ready.py index 7884767815..1fe3cefb5b 100644 --- a/build/ci/reconcile_deploy_ready.py +++ b/build/ci/reconcile_deploy_ready.py @@ -423,6 +423,36 @@ def _story_summary(story): return {"id": story.get("id"), "name": story.get("name"), "url": story.get("app_url")} +def _triage_context(story): + """Extra context for a TRIAGE story only, already sitting in the same + Shortcut search response that produced `story` -- adds no extra API + calls ("cheap", as opposed to e.g. an extra `gh pr view` round trip + per triage story, which this deliberately avoids). This is the raw + material a human (or the opt-in triage-explainer workflow step + downstream -- see build/ci/triage_explainer.py) needs to propose why a + story is stuck, without re-deriving it from scratch.""" + return { + "description": story.get("description"), + "comments": [c.get("text") for c in (story.get("comments") or []) if c.get("text")], + } + + +def _diagnose_linked_pr(pr, repo_id, target_branch): + """Which of the three PR-level guards (see qualifying_prs / + shortcut_pr_guards.passes_pr_guards) this specific linked PR fails, if + any. Diagnostic ONLY -- classification itself never reads this; it + exists purely so a triage story's report entry can say WHY a linked PR + didn't count instead of just listing its bare number.""" + failed = [] + if pr.get("merged") is not True: + failed.append("not merged") + if pr.get("repository_id") != repo_id: + failed.append(f"wrong repo (repository_id={pr.get('repository_id')}, expected {repo_id})") + if pr.get("target_branch_name") != target_branch: + failed.append(f"wrong target branch ({pr.get('target_branch_name')!r}, expected {target_branch!r})") + return failed + + def classify_stories(stories, repo_id, target_branch): """Pure classification, no I/O beyond what's already embedded in the Shortcut story payloads: split into (triage, candidates), where @@ -431,7 +461,12 @@ def classify_stories(stories, repo_id, target_branch): first and unconditionally routes to triage -- a story on any workflow other than Standard, or sitting at any state id other than the numeric Deploy Ready id (500000045) despite matching the "Deploy Ready" state - NAME search, must never reach the PR guards or a transition at all.""" + NAME search, must never reach the PR guards or a transition at all. + + Every triage entry also carries _triage_context (description, comment + text) -- shipped/pending entries deliberately do NOT, so they stay as + lean as before this was added; only a triage story's own report entry + ever needs to answer "why is this one stuck".""" triage = [] candidates = [] for story in stories: @@ -440,6 +475,7 @@ def classify_stories(stories, repo_id, target_branch): entry["reason"] = "non_standard_workflow_or_state" entry["workflow_id"] = story.get("workflow_id") entry["workflow_state_id"] = story.get("workflow_state_id") + entry.update(_triage_context(story)) triage.append(entry) warn( f"Story {story.get('id')} matched the Deploy Ready search but is on " @@ -457,6 +493,15 @@ def classify_stories(stories, repo_id, target_branch): entry["linked_pr_numbers"] = sorted( pr.get("number") for pr in linked if pr.get("number") is not None ) + # Full per-PR diagnostic, sorted the same way as + # linked_pr_numbers above for a stable, readable report -- + # linked_pr_numbers stays as-is (existing consumers rely on + # it); this is additive. + entry["linked_prs"] = [ + {"number": pr.get("number"), "failed_guards": _diagnose_linked_pr(pr, repo_id, target_branch)} + for pr in sorted(linked, key=lambda p: (p.get("number") is None, p.get("number"))) + ] + entry.update(_triage_context(story)) triage.append(entry) continue diff --git a/build/ci/tests/test_reconcile_deploy_ready.py b/build/ci/tests/test_reconcile_deploy_ready.py index 6016ec64c3..901db441c2 100644 --- a/build/ci/tests/test_reconcile_deploy_ready.py +++ b/build/ci/tests/test_reconcile_deploy_ready.py @@ -40,7 +40,7 @@ def _pr(number, merged=True, repository_id=SEFARIA_REPO_ID, target_branch_name=" def _story(story_id, name="Story", workflow_id=STANDARD_WORKFLOW_ID, workflow_state_id=DEPLOY_READY_STATE_ID, - pull_requests=None, branches=None): + pull_requests=None, branches=None, description=None, comments=None): return { "id": story_id, "name": name, @@ -49,6 +49,8 @@ def _story(story_id, name="Story", workflow_id=STANDARD_WORKFLOW_ID, workflow_st "workflow_state_id": workflow_state_id, "pull_requests": pull_requests or [], "branches": branches or [], + "description": description, + "comments": comments or [], } @@ -165,6 +167,93 @@ def test_classify_stories_no_qualifying_pr_routes_to_triage(): assert triage[0]["linked_pr_numbers"] == [3397] +# --- triage enrichment: description/comments/per-PR guard diagnostics --- +# --- (raw material for the opt-in triage-explainer step) ----------------- + +def test_diagnose_linked_pr_reports_every_failed_guard(): + pr = {"number": 1, "merged": False, "repository_id": 999, "target_branch_name": "preprod"} + failed = rdr._diagnose_linked_pr(pr, SEFARIA_REPO_ID, "master") + assert "not merged" in failed + assert any("wrong repo" in f for f in failed) + assert any("wrong target branch" in f for f in failed) + + +def test_diagnose_linked_pr_reports_no_failures_for_a_qualifying_pr(): + pr = _pr(3606) + assert rdr._diagnose_linked_pr(pr, SEFARIA_REPO_ID, "master") == [] + + +def test_diagnose_linked_pr_reports_only_the_specific_guard_that_failed(): + pr = _pr(3698, target_branch_name="preprod") # merged + right repo, wrong branch only + failed = rdr._diagnose_linked_pr(pr, SEFARIA_REPO_ID, "master") + assert failed == ["wrong target branch ('preprod', expected 'master')"] + + +def test_triage_context_extracts_description_and_comment_text(): + story = _story( + 11111, description="Some story description", + comments=[{"text": "first comment"}, {"text": "second comment"}, {"author_id": "x"}], + ) + ctx = rdr._triage_context(story) + assert ctx["description"] == "Some story description" + assert ctx["comments"] == ["first comment", "second comment"] + + +def test_triage_context_handles_missing_description_and_comments(): + story = {"id": 11111} + ctx = rdr._triage_context(story) + assert ctx["description"] is None + assert ctx["comments"] == [] + + +def test_classify_stories_no_qualifying_pr_entry_carries_full_context(monkeypatch): + """The raw material the opt-in explainer needs: description, comment + text, and -- per linked PR -- exactly which guard(s) it failed, not + just a bare number.""" + story = _story( + 11111, + pull_requests=[_pr(3698, target_branch_name="preprod"), _pr(3397, merged=False)], + description="A story description", + comments=[{"text": "why is this stuck?"}], + ) + triage, _ = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + entry = triage[0] + assert entry["description"] == "A story description" + assert entry["comments"] == ["why is this stuck?"] + assert entry["linked_pr_numbers"] == [3397, 3698] + by_number = {p["number"]: p["failed_guards"] for p in entry["linked_prs"]} + assert by_number[3698] == ["wrong target branch ('preprod', expected 'master')"] + assert by_number[3397] == ["not merged"] + + +def test_classify_stories_non_standard_workflow_entry_also_carries_context(): + story = _story( + 11111, workflow_id=500000061, workflow_state_id=500000900, + description="Lives on a different workflow", comments=[{"text": "not ours"}], + ) + triage, _ = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + entry = triage[0] + assert entry["description"] == "Lives on a different workflow" + assert entry["comments"] == ["not ours"] + + +def test_classify_stories_shipped_pending_candidates_do_not_carry_triage_context(monkeypatch): + """Shipped/pending stories must stay as lean as before this was added + -- _triage_context is triage-only, both by name and by application. + classify_stories only returns the raw (story, prs) pair for a + candidate; classify_candidates builds its own lean entry via + _story_summary, which never includes description/comments.""" + story = _story(11111, pull_requests=[_pr(3606)], description="should never appear") + triage, candidates = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") + assert triage == [] + assert len(candidates) == 1 + + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: True) + shipped, pending = rdr.classify_candidates(candidates, {3606: "abc"}, "prod/1.0") + assert "description" not in shipped[0] + assert "comments" not in shipped[0] + + def test_classify_stories_story_with_no_linked_prs_at_all_routes_to_triage(): story = _story(11111) triage, candidates = rdr.classify_stories([story], SEFARIA_REPO_ID, "master") diff --git a/build/ci/tests/test_triage_explainer.py b/build/ci/tests/test_triage_explainer.py new file mode 100644 index 0000000000..b7984139b2 --- /dev/null +++ b/build/ci/tests/test_triage_explainer.py @@ -0,0 +1,226 @@ +"""Tests for build/ci/triage_explainer.py. + +No network calls, no `git`/`gh` calls: this script is a plain, deterministic +JSON transform with no I/O beyond reading --report and writing --out. + +All story ids used below (11111, 22222, ...) are placeholders, not real +Shortcut story ids. +""" + +import json + +import pytest + +import triage_explainer as te + +FULL_REPORT = { + "prod_tag": "prod/7.1.3-prod.1+chart.0.88.2-prod.1", + "applied": False, + "counts": { + "total": 3, "shipped": 1, "pending": 1, "triage": 1, + "comment_posted": 0, "comment_failed": 0, + }, + "shipped": [ + {"id": 11111, "name": "Shipped story", "url": "https://app.shortcut.com/org/story/11111", + "shipped_via_prs": [3606], "qualifying_prs": [3606], "transitioned": True, + "would_comment": "should never leak into the explainer's input"}, + ], + "pending": [ + {"id": 22222, "name": "Pending story", "url": "https://app.shortcut.com/org/story/22222", + "qualifying_prs": [3670]}, + ], + "triage": [ + {"id": 33333, "name": "Triage story", "url": "https://app.shortcut.com/org/story/33333", + "reason": "no_qualifying_pr", "linked_pr_numbers": [3698], + "linked_prs": [{"number": 3698, "failed_guards": ["wrong target branch ('preprod', expected 'master')"]}], + "description": "A story description", "comments": ["why is this stuck?"]}, + ], + "comment_posted": [], + "comment_failed": [], +} + + +# --- extract_triage_only: the structural isolation guarantee ------------ + +def test_extract_triage_only_excludes_shipped_and_pending_entirely(): + result = te.extract_triage_only(FULL_REPORT) + assert "shipped" not in result + assert "pending" not in result + # Not just absent as top-level keys -- the shipped/pending story data + # itself (ids, names, would_comment text) must not appear anywhere in + # the serialized output. + serialized = json.dumps(result) + assert "11111" not in serialized + assert "22222" not in serialized + assert "should never leak" not in serialized + + +def test_extract_triage_only_excludes_applied_and_comment_bookkeeping(): + result = te.extract_triage_only(FULL_REPORT) + assert "applied" not in result + assert "comment_posted" not in result + assert "comment_failed" not in result + assert "counts" not in result # counts.shipped/pending would otherwise leak bucket sizes + + +def test_extract_triage_only_includes_triage_verbatim_with_full_context(): + result = te.extract_triage_only(FULL_REPORT) + assert result["triage"] == FULL_REPORT["triage"] + assert result["triage"][0]["description"] == "A story description" + assert result["triage"][0]["comments"] == ["why is this stuck?"] + assert result["triage"][0]["linked_prs"][0]["failed_guards"] == [ + "wrong target branch ('preprod', expected 'master')" + ] + + +def test_extract_triage_only_includes_prod_tag_and_triage_count(): + result = te.extract_triage_only(FULL_REPORT) + assert result["prod_tag"] == "prod/7.1.3-prod.1+chart.0.88.2-prod.1" + assert result["triage_count"] == 1 + + +def test_extract_triage_only_empty_triage_bucket(): + report = {**FULL_REPORT, "triage": []} + result = te.extract_triage_only(report) + assert result["triage"] == [] + assert result["triage_count"] == 0 + + +def test_extract_triage_only_missing_triage_key_degrades_to_empty(): + """A report from a hypothetical older/different reconcile run without a + 'triage' key at all must not crash -- it's just zero triage stories.""" + report = {"prod_tag": "prod/1.0"} + result = te.extract_triage_only(report) + assert result["triage"] == [] + assert result["triage_count"] == 0 + + +# --- main(): reads --report, writes --out --------------------------------- + +def test_main_reads_report_and_writes_triage_only_document(tmp_path, monkeypatch): + report_path = tmp_path / "reconcile-deploy-ready-report.json" + report_path.write_text(json.dumps(FULL_REPORT), encoding="utf-8") + out_path = tmp_path / "triage-only.json" + + monkeypatch.setattr( + "sys.argv", + ["triage_explainer.py", "extract", "--report", str(report_path), "--out", str(out_path)], + ) + te.main() + + data = json.loads(out_path.read_text(encoding="utf-8")) + assert "shipped" not in data + assert "pending" not in data + assert data["triage_count"] == 1 + assert data["triage"][0]["id"] == 33333 + + +def test_main_missing_report_file_dies_cleanly(tmp_path, monkeypatch): + monkeypatch.setattr( + "sys.argv", + ["triage_explainer.py", "extract", "--report", str(tmp_path / "does-not-exist.json"), + "--out", str(tmp_path / "out.json")], + ) + with pytest.raises(SystemExit): + te.main() + + +def test_main_malformed_report_json_dies_cleanly(tmp_path, monkeypatch): + report_path = tmp_path / "bad.json" + report_path.write_text("not valid json{{{", encoding="utf-8") + monkeypatch.setattr( + "sys.argv", + ["triage_explainer.py", "extract", "--report", str(report_path), "--out", str(tmp_path / "out.json")], + ) + with pytest.raises(SystemExit): + te.main() + + +# --- resolve_enabled: the opt-in decision, single-sourced and tested ---- +# --- ("disabled path" coverage) ------------------------------------------- + +def test_resolve_enabled_defaults_to_disabled_with_no_input_at_all(): + assert te.resolve_enabled("workflow_dispatch", "", "") is False + assert te.resolve_enabled("repository_dispatch", "", "") is False + + +def test_resolve_enabled_workflow_dispatch_true_input_enables(): + assert te.resolve_enabled("workflow_dispatch", "true", "") is True + + +def test_resolve_enabled_workflow_dispatch_ignores_the_repo_variable(): + """A workflow_dispatch run opts in via ITS OWN input only -- the + repo-level variable is for the repository_dispatch path and must never + leak in and silently enable a manual run that didn't ask for it.""" + assert te.resolve_enabled("workflow_dispatch", "", "true") is False + + +def test_resolve_enabled_repository_dispatch_true_var_enables(): + assert te.resolve_enabled("repository_dispatch", "", "true") is True + + +def test_resolve_enabled_repository_dispatch_ignores_the_workflow_dispatch_input(): + """The real automatic trigger carries no explain_triage input at all + (it's a repository_dispatch, not a workflow_dispatch) -- even if that + field were somehow non-empty, it must never be read on this path.""" + assert te.resolve_enabled("repository_dispatch", "true", "") is False + + +@pytest.mark.parametrize("value", ["false", "1", "yes", "True ", " true", "TRUE", "truex", None]) +def test_resolve_enabled_only_exact_true_enables(value): + # "TRUE" / "True " / " true" are still accepted (case/whitespace + # tolerant); everything else in this list is not. + result = te.resolve_enabled("workflow_dispatch", value, "") + if value is not None and value.strip().lower() == "true": + assert result is True + else: + assert result is False + + +def test_resolve_enabled_cli_prints_true_or_false(capsys, monkeypatch): + monkeypatch.setattr( + "sys.argv", + ["triage_explainer.py", "resolve-enabled", "--event-name", "workflow_dispatch", + "--explain-triage-input", "true", "--enable-var", ""], + ) + te.main() + assert capsys.readouterr().out.strip() == "true" + + +def test_resolve_enabled_cli_prints_false_when_disabled(capsys, monkeypatch): + monkeypatch.setattr( + "sys.argv", + ["triage_explainer.py", "resolve-enabled", "--event-name", "repository_dispatch", + "--explain-triage-input", "", "--enable-var", ""], + ) + te.main() + assert capsys.readouterr().out.strip() == "false" + + +# --- structural no-mutation / no-network guarantee ("NO mutation occurs -- +# --- on any path" coverage for the Python side of this feature) ---------- + +def test_triage_explainer_module_never_imports_network_or_subprocess(): + """Architectural guarantee, not just a convention: this script is a + pure JSON transform plus a pure string-comparison decision function. It + must never import urllib, requests, subprocess, or anything else that + could reach a network or mutate external/process state -- the actual + LLM call (a `claude -p` CLI invocation) and any Shortcut mutation stay + entirely in the opt-in workflow step and the deterministic + reconcile_deploy_ready.py / mark_stories_deployed.py scripts, never + here. Checked by inspecting this module's own top-level imports + directly, not the source text (so a forbidden name appearing inside a + string or comment can't cause a false failure).""" + import ast + import inspect + + tree = ast.parse(inspect.getsource(te)) + imported_names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_names.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported_names.add(node.module.split(".")[0]) + + forbidden = {"urllib", "requests", "subprocess", "socket", "http"} + assert not (imported_names & forbidden), imported_names & forbidden diff --git a/build/ci/triage_explainer.py b/build/ci/triage_explainer.py new file mode 100644 index 0000000000..538f5cf0cc --- /dev/null +++ b/build/ci/triage_explainer.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +""" +Extract JUST the triage bucket from a reconcile_deploy_ready.py report, +plus the minimal safe context an explainer needs, into a separate, +structurally isolated document. + +Today reconcile_deploy_ready.py's triage bucket is reported as +`reason=no_qualifying_pr` (or `non_standard_workflow_or_state`) plus raw +diagnostic fields, which leaves a human to open every story and work out +each one individually. An OPT-IN workflow step +(.github/workflows/prod-release-notes.yaml) runs a headless `claude -p` +over exactly the document this script produces to propose a short +hypothesis and a suggested next action per triage story -- but the LLM +call itself lives entirely in that workflow step, never here. This script +is a plain, deterministic JSON transform: no API client, no `claude` +invocation, no network. Keeping reconcile_deploy_ready.py (and this +sibling script) stdlib-only and LLM-free, with the model's role confined +to writing English from a document it's handed, is the same design +principle the rest of this pipeline already follows (see +reconcile_deploy_ready.py's and shipped_stories.py's own docstrings). + +Why a SEPARATE document rather than just telling the model "only look at +the triage section" of the full report: the full report's `shipped` and +`pending` buckets, and reconcile_deploy_ready.py's own JSON write-back +comment content, must never be visible to, scored by, or able to +influence this explainer -- and a prompt instruction is not a security +boundary against a prompt-injection path. A triage story's `description` +and `comments` fields are CONTRIBUTOR-CONTROLLED TEXT (see the workflow +step's own comment for why that scopes its allowed tools). Rather than +trust the model to honor "ignore the other buckets" against adversarial +input embedded in the very document it's reading, this script simply never +puts shipped/pending data into the file the explainer is given at all -- +there is nothing there to leak or be steered by, structurally, not merely +by convention. + +This script also owns the opt-in DECISION (resolve_enabled / +`resolve-enabled` subcommand) for the same single-source-of-truth reason: +without it, "is the explainer enabled" would be a small bash string +comparison duplicated (and possibly drifted) inline in the workflow YAML, +untested by anything. Putting it here means the workflow's `run:` block +just calls this script and the actual rule lives in one tested place. + +Usage: + python3 triage_explainer.py extract --report reconcile-deploy-ready-report.json --out triage-only.json + python3 triage_explainer.py resolve-enabled --event-name workflow_dispatch --explain-triage-input true --enable-var "" + +All story ids in this file's docstring and comments (e.g. 11111) are +placeholders, not real Shortcut story ids. +""" + +import argparse +import json +import sys + +# Keys from a reconcile_deploy_ready.py report that this script MUST NEVER +# copy into its output, even if a future edit to that report adds new +# top-level keys carelessly. Listed explicitly (rather than an +# allow-only-"triage" approach implemented by construction below) as a +# second, redundant line of defense -- see extract_triage_only. +EXCLUDED_REPORT_KEYS = frozenset({ + "shipped", "pending", "applied", "comment_posted", "comment_failed", +}) + + +def die(message: str) -> None: + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def extract_triage_only(report): + """Build the explainer's ENTIRE input: the triage list verbatim, as + reconcile_deploy_ready.py already enriched each entry (description, + comments, linked_prs with per-PR guard diagnostics -- see + reconcile_deploy_ready.classify_stories / _triage_context / + _diagnose_linked_pr), plus `prod_tag` (so an explanation can say "as of + " without guessing) and a `triage_count` the caller can check + cheaply to skip the whole explainer step when there's nothing to + explain. This is constructed as an explicit allow-list (only these + three keys are ever read from `report` and copied out) rather than + "copy everything except EXCLUDED_REPORT_KEYS" -- an allow-list can't + accidentally leak a new field a future report format adds; a + deny-list could.""" + triage = report.get("triage") or [] + result = { + "prod_tag": report.get("prod_tag"), + "triage_count": len(triage), + "triage": triage, + } + # Defensive, should be unreachable given the allow-list above -- kept + # as a loud assertion rather than silently trusting the allow-list + # forever stays correct. + leaked = EXCLUDED_REPORT_KEYS & result.keys() + if leaked: + die(f"internal error: triage-only extraction would have leaked {sorted(leaked)} -- refusing to write it") + return result + + +def resolve_enabled(event_name, explain_triage_input, enable_var): + """Whether the opt-in triage explainer should run this trigger. Pure + decision logic, no I/O -- both inputs default to disabled on any falsy + or unrecognized value, never enabled by omission or by an unexpected + string: + + - A `workflow_dispatch` run opts in per-run via its own + `explain_triage` boolean input. + - The `repository_dispatch` trigger (the real automatic + post-promotion path -- see the workflow header) carries no such + input at all, so it instead opts in via a repo-level Actions + variable (`vars.ENABLE_TRIAGE_EXPLAINER`), which a human sets + independently of any single run. + + Only an exact case-insensitive "true" enables anything; every other + value (empty, "false", "1", a typo, ...) is treated as disabled. This + means the feature ships fully OFF by default on every trigger path + until a human explicitly flips one of the two switches.""" + value = explain_triage_input if event_name == "workflow_dispatch" else enable_var + return str(value).strip().lower() == "true" + + +def build_arg_parser(): + parser = argparse.ArgumentParser( + description="Extract the triage bucket from a reconcile_deploy_ready.py report, or " + "resolve whether the opt-in triage explainer should run -- for the " + "opt-in triage-explainer workflow step.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + extract_parser = subparsers.add_parser( + "extract", help="Write the triage-only document for a reconcile report to --out.", + ) + extract_parser.add_argument("--report", required=True, help="Path to a reconcile_deploy_ready.py --out report JSON") + extract_parser.add_argument("--out", required=True, help="Output path for the triage-only document") + + enabled_parser = subparsers.add_parser( + "resolve-enabled", + help="Print 'true' or 'false' to stdout: should the explainer run for this trigger?", + ) + enabled_parser.add_argument("--event-name", required=True, help="github.event_name, e.g. workflow_dispatch") + enabled_parser.add_argument("--explain-triage-input", default="", help="inputs.explain_triage (workflow_dispatch only)") + enabled_parser.add_argument("--enable-var", default="", help="vars.ENABLE_TRIAGE_EXPLAINER") + + return parser + + +def main(): + args = build_arg_parser().parse_args() + + if args.command == "resolve-enabled": + print("true" if resolve_enabled(args.event_name, args.explain_triage_input, args.enable_var) else "false") + return + + # command == "extract" + try: + with open(args.report, encoding="utf-8") as f: + report = json.load(f) + except (OSError, json.JSONDecodeError) as e: + die(f"Could not read/parse --report {args.report!r}: {e}") + + triage_only = extract_triage_only(report) + + with open(args.out, "w", encoding="utf-8") as f: + json.dump(triage_only, f, indent=2, ensure_ascii=False) + f.write("\n") + + +if __name__ == "__main__": + main() From 37e06882e76c9e7cdeaad8abd17d011090f09de8 Mon Sep 17 00:00:00 2001 From: Yotam Fromm Date: Mon, 7 Sep 2026 10:34:53 +0300 Subject: [PATCH 4/6] fix(ci): drop write-back comments, add head-branch PR guard, backfill current-release stories into notes Three changes to the Deploy Ready reconciliation pipeline: 1. Remove the Shortcut write-back comment feature added previously (product decision: transitioning to Done is enough, no per-story comment needed). Deletes shortcut_comment.py, the comment-text builders, --no-comment flags, and comment_posted/comment_failed/ would_comment fields from both mark_stories_deployed.py and reconcile_deploy_ready.py. resolve_shipping_release_tag() is kept -- change 3 below needs it. 2. Fix a guard drift in the opposite direction from the one fixed previously: reconcile_deploy_ready.py had no guard against a PR whose HEAD branch is a long-lived environment branch (a promotion PR merging preprod -> master passes merged/repo/target-branch cleanly, since its target really is master). shipped_stories.py's RC1 fallback already had this check for its own purposes; the two scripts drifted apart on whether it existed at all. Unified as a fourth PR-level guard in the shared shortcut_pr_guards.py, applied by both callers. 3. A story the reconciliation sweep backfills can, in the ordinary case, have shipped in the CURRENT release rather than an earlier one -- silently excluding it from today's announcement was a different flavor of the same "say what actually shipped today" mistake, just by omission instead of leakage. reconcile_deploy_ready.py now attaches shipping_release_tag (derived via resolve_shipping_release_tag) and, only when it equals the current prod tag, a hydrated_story sub-object to each shipped report entry. A new, separate, deterministic script (merge_release_backfill.py) trusts that upstream decision and folds only those pre-filtered entries into shipped-stories.json, deduped by id, before the prose step runs -- the reconcile report's shipped/ pending/triage buckets as a whole still never reach that step. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KJqBksQqzHYs3F54Y4tJRX --- .github/workflows/prod-release-notes.yaml | 79 +++- build/ci/README-prod-release-notes.md | 274 +++++++------- build/ci/mark_stories_deployed.py | 121 +----- build/ci/merge_release_backfill.py | 170 +++++++++ build/ci/reconcile_deploy_ready.py | 253 ++++++------- build/ci/shipped_stories.py | 37 +- build/ci/shortcut_comment.py | 66 ---- build/ci/shortcut_pr_guards.py | 41 ++- build/ci/tests/test_mark_stories_deployed.py | 215 +---------- build/ci/tests/test_merge_release_backfill.py | 261 +++++++++++++ build/ci/tests/test_reconcile_deploy_ready.py | 348 +++++++++--------- build/ci/tests/test_shipped_stories.py | 54 ++- build/ci/tests/test_triage_explainer.py | 17 +- build/ci/triage_explainer.py | 4 +- 14 files changed, 1051 insertions(+), 889 deletions(-) create mode 100644 build/ci/merge_release_backfill.py delete mode 100644 build/ci/shortcut_comment.py create mode 100644 build/ci/tests/test_merge_release_backfill.py diff --git a/.github/workflows/prod-release-notes.yaml b/.github/workflows/prod-release-notes.yaml index dda279c360..def94a4fe3 100644 --- a/.github/workflows/prod-release-notes.yaml +++ b/.github/workflows/prod-release-notes.yaml @@ -8,39 +8,47 @@ name: "Prod Release Notes" # # From there this workflow is a strict pipeline of deterministic scripts, with # two steps handed to an LLM (release-notes prose, and an OPT-IN triage -# explanation -- see step 4 below): +# explanation -- see step 5 below): # 1. build/ci/shipped_stories.py resolves the prod/* tag range for that # version (chartVersion disambiguates a chart-only rollout, where several # tags share the same app version), walks git log between the tags, and # hydrates the Shortcut stories those commits reference. # 2. build/ci/mark_stories_deployed.py moves each of those stories from -# Deploy Ready -> Done via the Shortcut API, then posts a write-back -# comment on each naming this release. A failure here (missing token, -# API error, nothing to move) is surfaced as a warning but never blocks -# steps 3-6 — see "Mark shipped stories as deployed" below. +# Deploy Ready -> Done via the Shortcut API. It only ever writes that +# state -- no comment, no other annotation. A failure here (missing +# token, API error, nothing to move) is surfaced as a warning but never +# blocks steps 3-7 — see "Mark shipped stories as deployed" below. # 3. build/ci/reconcile_deploy_ready.py sweeps EVERY non-archived Deploy # Ready story org-wide (not just this release's commit range) and # transitions any whose linked PR already reached prod, regardless of -# which release actually shipped it, then posts its own write-back -# comment naming the release that ACTUALLY shipped it (never this one). -# This backfills stories a prior release's git-range-scoped run never -# revisited (RC2) — see that script's own docstring for the four guards -# it applies. Its output NEVER feeds shipped-stories.json or steps 5-6 -# below: those stories shipped in EARLIER releases, and leaking them -# into today's announcement would have Slack claim old features shipped -# today. A failure here is warned/alerted the same way step 2's is, and -# never blocks steps 4-6. -# 4. build/ci/triage_explainer.py + a headless `claude -p` run (OPT-IN, +# which release actually shipped it -- same "state only" posture as +# step 2. This backfills stories a prior release's git-range-scoped run +# never revisited (RC2) — see that script's own docstring for the four +# PR-level guards (+ one story-level workflow/state guard) it applies. +# Its report is written to $RUNNER_TEMP and NEVER read directly by +# steps 6-7: most of what it backfills shipped in EARLIER releases, and +# leaking that wholesale into today's announcement would have Slack +# claim old features shipped today. A failure here is warned/alerted +# the same way step 2's is, and never blocks steps 4-7. +# 4. build/ci/merge_release_backfill.py folds the ONE, pre-filtered +# exception into shipped-stories.json: a story step 3 backfilled whose +# TRUE shipping release (resolved there, from git) equals THIS run's +# own current release, not an earlier one -- that story really did +# just ship today and belongs in the announcement. This is the only +# channel through which anything from step 3's report ever reaches +# shipped-stories.json, and the decision was already made +# deterministically upstream; this step is a pure mechanical merge. +# 5. build/ci/triage_explainer.py + a headless `claude -p` run (OPT-IN, # off by default -- see "Explain Deploy Ready triage backlog" below) # proposes a short, labeled HYPOTHESIS for why each story in step 3's # triage bucket is stuck, and a suggested next action for a human. It # sees ONLY the triage bucket (never shipped/pending, structurally — # see triage_explainer.py's own docstring), never mutates Shortcut, -# and its output never reaches steps 5-6 either. -# 5. The sefaria-release-notes skill (.claude/skills/sefaria-release-notes/) -# reads that JSON and writes prose only — it does not talk to GitHub or -# Shortcut. -# 6. scripts/post_to_slack.py posts both generated files to Slack. +# and its output never reaches steps 6-7 either. +# 6. The sefaria-release-notes skill (.claude/skills/sefaria-release-notes/) +# reads shipped-stories.json (now including step 4's merge, if any) and +# writes prose only — it does not talk to GitHub or Shortcut. +# 7. scripts/post_to_slack.py posts both generated files to Slack. # # Manual setup this depends on: see build/ci/README-prod-release-notes.md @@ -255,6 +263,37 @@ jobs: --data "{\"text\": \":warning: Prod release notes for version ${VERSION:-unknown}: ${FAILURES}Release notes generation is continuing. Check the workflow run and move the affected stories manually.\"}" \ "$SLACK_WEBHOOK_URL" || true + - name: Merge current-release backfill into shipped-stories.json + id: merge_backfill + # Runs AFTER "Reconcile Deploy Ready backlog" writes its report, + # and BEFORE "Generate release notes" reads shipped-stories.json -- + # order is load-bearing here. Folds in the ONE, already-decided + # exception to "the reconcile report never reaches the prose step": + # a story that sweep backfilled whose derived TRUE shipping release + # (git evidence, resolved in reconcile_deploy_ready.py itself) is + # THIS run's own current release, not an earlier one. See + # merge_release_backfill.py's own docstring for why trusting that + # upstream decision, rather than re-deciding anything here, is what + # keeps this a pure mechanical merge. Reads the reconcile report + # from $RUNNER_TEMP (never the checkout) and only ever WRITES + # shipped-stories.json -- the report itself is still never handed + # to, or made readable by, the prose step below. + # + # continue-on-error: a failure here means, at worst, one backfilled + # story misses today's announcement (an omission, not a leak) -- + # never worth blocking release-notes generation and Slack posting + # for an otherwise healthy rollout, same posture as steps 2-3 above. + continue-on-error: true + run: | + REPORT_FILE="$RUNNER_TEMP/reconcile-deploy-ready-report.json" + if [[ ! -s "$REPORT_FILE" ]]; then + echo "No reconcile report found at $REPORT_FILE (the reconcile step may have failed or produced nothing) -- nothing to merge." + exit 0 + fi + python3 build/ci/merge_release_backfill.py \ + --shipped-stories-out shipped-stories.json \ + --reconcile-report "$REPORT_FILE" + - name: Resolve triage explainer opt-in id: triage_opt_in env: diff --git a/build/ci/README-prod-release-notes.md b/build/ci/README-prod-release-notes.md index 64c76b1273..f8e5dce696 100644 --- a/build/ci/README-prod-release-notes.md +++ b/build/ci/README-prod-release-notes.md @@ -17,23 +17,32 @@ Argo post-promotion analysis (prod) link, hydrates story details -> build/ci/mark_stories_deployed.py — moves each shipped story Deploy Ready -> Done via the - Shortcut API, then posts a write-back - comment naming this release. A - failure here (missing token, API - error, nothing to move) is logged and - Slack-alerted but never blocks the - steps below. + Shortcut API. Only ever writes that + state — no comment, no other + annotation. A failure here (missing + token, API error, nothing to move) is + logged and Slack-alerted but never + blocks the steps below. -> build/ci/reconcile_deploy_ready.py — separately, sweeps EVERY non-archived Deploy Ready story org-wide (not just this release's commit range) and transitions any - whose linked PR already reached prod, - then posts its own write-back comment - naming the release that ACTUALLY - shipped it (never this one) — see - "Reconciliation sweep" and "Write-back - release comment" below. Its output - NEVER reaches the steps that follow. + whose linked PR already reached prod + — see "Reconciliation sweep" below. + Its report is written to + `$RUNNER_TEMP` and NEVER read + wholesale by the steps that follow. + -> build/ci/merge_release_backfill.py — folds the ONE, pre-filtered + exception into shipped-stories.json: + a story the sweep backfilled whose + derived TRUE shipping release is + THIS run's own current release, not + an earlier one — see "Backfilled + stories that shipped in THIS + release" below. Only ever touches + shipped-stories.json; never reads + the reconcile report's shipped/ + pending/triage buckets wholesale. -> build/ci/triage_explainer.py — OPT-IN, off by default: extracts + headless `claude -p` ONLY the triage bucket into its own file, then proposes a labeled @@ -42,8 +51,9 @@ Argo post-promotion analysis (prod) explainer" below. Never mutates Shortcut; its output NEVER reaches the two steps that follow either. - -> sefaria-release-notes skill — reads shipped-stories.json, writes - prose only + -> sefaria-release-notes skill — reads shipped-stories.json (now + including the merge step's backfill, + if any), writes prose only -> scripts/post_to_slack.py — posts both files to Slack ``` @@ -86,9 +96,10 @@ order: merge/branch-sync noise (`NOISE_PATTERN` — the same pattern already used to keep such commits out of `commits_without_story`) or whose PR's own head branch is a long-lived environment branch, and (b) re-checks the - single search result's own linked-PR entry against the three PR-level - guards (merged / Sefaria-Project repo / target branch master) before - adopting it — a match that fails those guards is a warn-and-skip. + single search result's own linked-PR entry against the four PR-level + guards (merged / Sefaria-Project repo / target branch master / head + branch not a long-lived environment branch) before adopting it — a + match that fails those guards is a warn-and-skip. Ids recovered this way are echoed separately in the output's `stories_from_shortcut_pr_link` list (in @@ -121,17 +132,18 @@ story into exactly one of three buckets: non-Standard Shortcut workflow. Left alone and reported; this is the part of the output a human actually has to look at. -### The four guards +### The five guards -Each of these caught a real false positive while this script was verified -against live data — skipping any one of them silently mis-transitions a -story. The first three (merged / repo / target branch) are PR-level checks -shared with `shipped_stories.py`'s own RC1 PR-link fallback via -`build/ci/shortcut_pr_guards.py` — both scripts ask the same underlying -question ("does this linked PR actually prove a story's change reached -prod?") and a promotion PR is exactly as good at fooling either one, so -there is exactly one implementation of these three checks, not two -parallel copies that could silently drift apart: +Each of these caught a real false positive while this script was built (or +maintained) against live data — skipping any one of them silently +mis-transitions a story. The first four (merged / repo / target branch / +head branch) are PR-level checks shared with `shipped_stories.py`'s own +RC1 PR-link fallback via `build/ci/shortcut_pr_guards.py` — both scripts +ask the same underlying question ("does this linked PR actually prove a +story's change reached prod?") and a promotion PR is exactly as good at +fooling either one, so there is exactly one implementation of these four +checks, not two parallel copies that could silently drift apart. They +already have, twice, in OPPOSITE directions: 1. **`repository_id` must be Sefaria-Project's (`500000103`).** A story can link a PR from a different repo; resolving that PR number against @@ -143,9 +155,23 @@ parallel copies that could silently drift apart: promotion PR (preprod -> prod, or master -> preprod) instead of, or alongside, the actual feature PR. A promotion PR merges constantly and proves nothing about whether this story's own change reached prod. -3. **`merged` must be `true`.** An open or closed-without-merging PR is not +3. **The PR's own HEAD (source) branch must NOT be a long-lived + environment branch** (`master`, `preprod`, `prod`). Verified live: a + promotion PR merging `preprod` INTO `master` passes guards 1, 2 and 4 + cleanly — it's merged, against the right repo, and its target really is + `master` — yet it's still a promotion merge, not a feature PR. Guard 2 + alone cannot catch this shape (a promotion merge legitimately targets + `master`); only the HEAD branch gives it away. **This is the guard that + was missing** when a real story was classified `shipped` on the + strength of a PR shaped exactly this way, while its genuine feature PR + was (correctly) rejected by guard 2 for targeting a hotfix branch + instead of `master` directly. `shipped_stories.py`'s RC1 fallback + already had this check for its own purposes before `reconcile_ + deploy_ready.py` did — the two scripts drifted apart on whether this + guard existed at all before it was unified here. +4. **`merged` must be `true`.** An open or closed-without-merging PR is not evidence anything shipped. -4. **`workflow_id` must be the Standard workflow (`500000005`), and +5. **`workflow_id` must be the Standard workflow (`500000005`), and `workflow_state_id` must be exactly the numeric Deploy Ready id (`500000045`).** The Shortcut state named "Deploy Ready" — note its real name carries a trailing space, `"Deploy Ready "` — is workflow-specific: @@ -155,7 +181,8 @@ parallel copies that could silently drift apart: trusting a match; a story on any other workflow, or at any other state id despite matching the name, is routed to triage with its actual workflow/state ids reported — mirroring `mark_stories_deployed.py`'s - `skipped_different_workflow` handling. + `skipped_different_workflow` handling. This one is story-level, not + PR-level, and stays local to `reconcile_deploy_ready.py`. Enumeration uses the token'd search endpoint (`search/stories?query=state:"Deploy Ready" !is:archived`), paginated via @@ -190,89 +217,90 @@ python3 build/ci/reconcile_deploy_ready.py --apply # actually python3 build/ci/reconcile_deploy_ready.py --apply --prod-tag prod/6.111.0-prod.2+chart.0.87.5-prod.1 --out report.json ``` -**Critical: this script never reads or writes `shipped-stories.json` and -never feeds the release-notes prose step.** The stories it backfills -shipped in EARLIER releases — leaking them into today's release -announcement would have Slack claim a dozen old features shipped today. -Reconciliation transitions Shortcut state only; it has no opinion about -what today's release notes should say. In the workflow, its step runs -after `mark_stories_deployed.py` and writes its own separate report file -to `$RUNNER_TEMP` (NOT the checkout / `GITHUB_WORKSPACE`) — a distinct -filename alone is a naming convention, not an access boundary, and the -release-notes step's headless Claude run holds `Glob`+`Read` over its whole -working directory, so a same-directory JSON full of real, recently-shipped -story names would be one bad glob away from leaking into the prose it -writes. Keeping the report outside the checkout entirely is what actually -enforces the separation. A failure here is warned/Slack-alerted the same way a -`mark_stories_deployed.py` failure is, and never blocks release-notes -generation or posting. - -## Write-back release comment - -Until this feature, the pipeline only ever WROTE a story's workflow state -(the `PUT` that moves it Deploy Ready -> Done) — nothing recorded WHICH -release actually carried a story, so a person reading it in Shortcut could -see it became Done but had no way to tell what shipped it without going and -digging through CI logs or git. Both `mark_stories_deployed.py` and -`reconcile_deploy_ready.py` now post a short, factual comment -(`POST /stories/{id}/comments`) immediately after a story is ACTUALLY -transitioned by that run — never for already-Done/skipped stories, and -never merely because a dry-run run classified something as a candidate. - -The POST mechanics (`build/ci/shortcut_comment.py`) are shared between the -two scripts for the same drift-prevention reason `shortcut_pr_guards.py` -is shared for the PR-level guards — one implementation, not two copies that -could quietly diverge. The comment TEXT is deliberately **not** shared, -because the two scripts know different things about which release actually -shipped a story: - -- **`mark_stories_deployed.py`** is reading THIS release's own - shipped-stories.json, so it already has `version`, `chart_version` and - `release_date` for exactly the release a story just shipped in — the - comment states that directly, plus the PR(s) that carried it (resolved - from the JSON's own `commits` list). -- **`reconcile_deploy_ready.py`** does NOT know that — a story it backfills - shipped in some EARLIER release, and if its comment named the CURRENT - prod tag, a reader would reasonably conclude that story shipped in - TODAY's release. That is exactly the "old features shipped today" error - class the `shipped-stories.json` / reconcile-report separation already - documented above exists to prevent — just showing up in a Shortcut - comment instead of a Slack post. So it instead asks git for the TRUE - release: - - ``` - git tag --list 'prod/*' --contains --sort=creatordate | head -1 - ``` - - the first (earliest-created) `prod/*` tag that actually contains the - winning PR's merge commit — the release that really carried it. Note the - ascending `--sort=creatordate` here, the OPPOSITE of - `resolve_default_prod_tag`'s `-creatordate`: that one wants the newest - tag (today's release); this one wants the OLDEST tag that still contains - the commit, i.e. the first release it ever reached. If that lookup can't - be resolved for any reason (shallow checkout, a genuine gap in tag - history, ...), the comment degrades HONESTLY — it says only that the - story was detected as already present in production as of the current - prod tag, and names the PR. It never guesses or implies a specific - release. - -Safety properties, both scripts: - -- Posted ONLY after a transition actually succeeds; a failed transition - posts nothing. -- Never posted in `--dry-run` (or reconcile's default no-`--apply` mode) — - the report instead shows what WOULD be posted (`would_comment` in the - JSON, a preview line in the stdout summary). -- A comment failure is logged (`WARNING` to stderr) and recorded - (`comment_failed` in the JSON summary/report) but never fails the run or - rolls back the already-successful transition — the state change is the - valuable, already-durable part; the comment is a best-effort annotation - on top of it. -- `--no-comment` on both scripts opts out of the annotation entirely while - still transitioning. -- No separate dedupe index: a transitioned story leaves Deploy Ready, so a - re-run's search/classify simply never sees it again — idempotency falls - out of the state machine for free. +**Critical: this script never reads or writes `shipped-stories.json` itself +and never talks to the release-notes prose step directly.** Most of what it +backfills shipped in EARLIER releases — leaking that wholesale into today's +release announcement would have Slack claim a dozen old features shipped +today. This script only ever transitions Shortcut state; it never posts a +comment or any other annotation ("just mark it as done" is the whole job +here). In the workflow, its step runs after `mark_stories_deployed.py` and +writes its own separate report file to `$RUNNER_TEMP` (NOT the checkout / +`GITHUB_WORKSPACE`) — a distinct filename alone is a naming convention, not +an access boundary, and the release-notes step's headless Claude run holds +`Glob`+`Read` over its whole working directory, so a same-directory JSON +full of real, recently-shipped story names would be one bad glob away from +leaking into the prose it writes. Keeping the report outside the checkout +entirely is what actually enforces the separation. A failure here is +warned/Slack-alerted the same way a `mark_stories_deployed.py` failure is, +and never blocks release-notes generation or posting. + +One narrow, explicitly-filtered exception to "never feeds the prose step" +exists — see the next section. + +## Backfilled stories that shipped in THIS release: `build/ci/merge_release_backfill.py` + +The reconciliation sweep exists because nothing else revisits a Deploy +Ready story once it falls outside the current release's git-range scan. +Most of what it finds shipped in an EARLIER release, and the previous +section is about keeping THAT out of today's announcement. But some of +what it finds shipped in the CURRENT release too — a race, a discovery gap +`shipped_stories.py`'s own RC1 fallback didn't close, a story that simply +never got picked up by `mark_stories_deployed.py`'s own commit-range scan. +Silently excluding that story from today's announcement is a different +flavor of the same underlying mistake ("say what actually shipped today"), +just by omission instead of leakage. + +Every entry in `reconcile_deploy_ready.py`'s `shipped` bucket now carries a +`shipping_release_tag` field: the TRUE release that shipped it, resolved +via `resolve_shipping_release_tag()` (`git tag --list 'prod/*' --contains + --sort=creatordate | head -1` — the first, earliest-created, +`prod/*` tag that actually contains the winning PR's merge commit). Note +the ascending `--sort=creatordate` — the OPPOSITE of the default-prod-tag +resolution elsewhere in this script, which wants the NEWEST tag; this one +wants the OLDEST tag that still contains the commit, i.e. the first +release it ever reached. When `shipping_release_tag` equals the CURRENT +prod tag that run was checking against, the entry ALSO carries a +`hydrated_story` sub-object — a story record in exactly +`shipped_stories.py`'s own hydrated-story shape (`id`, `name`, +`description`, `url`, `workflow_id`, `workflow_state_id`, `story_type`), +built from data the sweep already had in memory (no extra Shortcut API +call). When `resolve_shipping_release_tag()` can't determine a release at +all (shallow checkout, a genuine gap in tag history), the entry gets +NEITHER field populated with a guess — fail closed, never guess a story +into an announcement. + +`merge_release_backfill.py` is a separate, tiny, deterministic script that +trusts that upstream decision completely and makes no judgment of its own: +it reads `reconcile_deploy_ready.py`'s report, takes ONLY the +`hydrated_story` entries (every other shipped entry — earlier release, +unresolvable release, or a pending/triage entry — carries no such field +and is silently skipped), and folds them into `shipped-stories.json`'s own +`story_ids`/`stories` lists, deduplicated by story id against what +`shipped_stories.py`'s own git-range + RC1 discovery already found (a +story both paths independently find is never duplicated; the pre-existing +entry always wins). The merged ids are also recorded separately under +`stories_from_reconciliation_backfill`, mirroring `shipped_stories.py`'s +own `stories_from_shortcut_pr_link` provenance field. + +``` +python3 build/ci/merge_release_backfill.py \ + --shipped-stories-out shipped-stories.json \ + --reconcile-report reconcile-deploy-ready-report.json + # writes the merged result back to --shipped-stories-out by default; + # pass --out to write somewhere else instead. +``` + +In the workflow, this step runs immediately after `reconcile_deploy_ready.py` +and BEFORE the release-notes prose step — order is load-bearing, since the +prose step reads `shipped-stories.json` once and needs the merge already +done. It reads the reconcile report from `$RUNNER_TEMP` (never the +checkout) and only ever WRITES `shipped-stories.json` — the report's +`shipped`/`pending`/`triage` buckets as a whole are still never handed to, +or made readable by, the prose step; only the pre-filtered +`hydrated_story` entries this script extracts ever reach it. A failure +here means, at worst, one backfilled story misses today's announcement (an +omission, not a leak) — never worth blocking release-notes generation and +Slack posting over, so this step is `continue-on-error` too. ## Opt-in triage explainer @@ -327,10 +355,10 @@ below**, on purpose: - Every hypothesis string is required (by the prompt) to start with the literal `[AI hypothesis, unverified]` marker, so it can never be mistaken for a verified finding by whoever reads it. -- **Proposes, never decides**: no Shortcut mutation, no comment posted - from its output, no transition — of ANY kind, on ANY story. Authority - stays entirely with the deterministic layer (the four guards, the - ancestry check) and the human reading the report. An ambiguous `pr:` +- **Proposes, never decides**: no Shortcut mutation, no transition — of + ANY kind, on ANY story. Authority stays entirely with the deterministic + layer (the five guards, the ancestry check) and the human reading the + report. An ambiguous `pr:` lookup returning more than one story is still a deterministic warn-and-skip in `shipped_stories.py`/`reconcile_deploy_ready.py` (unchanged) — that's precisely the "model decides two things are @@ -495,10 +523,10 @@ anywhere — it would just be quiet. The tests for these scripts (`build/ci/tests/test_shipped_stories.py`, `test_mark_stories_deployed.py`, `test_reconcile_deploy_ready.py`, -`test_triage_explainer.py`) are **not** collected by the repo's root -`pytest.ini` (that config is scoped to the Django app's own test suites), -so run them by explicit path from the repo root — either the whole -directory: +`test_triage_explainer.py`, `test_merge_release_backfill.py`) are **not** +collected by the repo's root `pytest.ini` (that config is scoped to the +Django app's own test suites), so run them by explicit path from the repo +root — either the whole directory: ``` python3 -m pytest build/ci/tests/ -q -p no:django -c /dev/null @@ -507,7 +535,7 @@ python3 -m pytest build/ci/tests/ -q -p no:django -c /dev/null or each file explicitly: ``` -python3 -m pytest build/ci/tests/test_shipped_stories.py build/ci/tests/test_mark_stories_deployed.py build/ci/tests/test_reconcile_deploy_ready.py build/ci/tests/test_triage_explainer.py -q -p no:django -c /dev/null +python3 -m pytest build/ci/tests/test_shipped_stories.py build/ci/tests/test_mark_stories_deployed.py build/ci/tests/test_reconcile_deploy_ready.py build/ci/tests/test_triage_explainer.py build/ci/tests/test_merge_release_backfill.py -q -p no:django -c /dev/null ``` Both flags are needed even though only explicit file paths are passed: diff --git a/build/ci/mark_stories_deployed.py b/build/ci/mark_stories_deployed.py index 57c73d1add..cdf329ee67 100755 --- a/build/ci/mark_stories_deployed.py +++ b/build/ci/mark_stories_deployed.py @@ -36,22 +36,6 @@ silently lose stories that shipped but that shipped_stories.py could not look up. -Immediately after a story is ACTUALLY transitioned by this run (never for -already_done/skipped stories, and never merely because it was a candidate), -a short write-back comment is posted on it via -`POST /stories/{id}/comments` (shortcut_comment.py, shared with -reconcile_deploy_ready.py's own write-back) naming the release that shipped -it: this run's own --input JSON already carries `version`, `chart_version` -and `release_date` for exactly that release, so there's no ambiguity to -guard against here the way reconcile_deploy_ready.py's backfill sweep has -to (see that script's own docstring). The PR(s) that carried the story, if -resolvable from --input's `commits` list, are included as GitHub links. -A comment failure is logged and recorded in `comment_failed` but never -fails the run or rolls back the transition -- the state change is the -valuable, already-durable part; the comment is a best-effort annotation on -top of it. Posting is skipped entirely in --dry-run (which instead prints -what WOULD be posted) and with --no-comment. - Whenever the input's "stories" list is non-empty and at least one story was skipped for a reason other than already being Done, this script prints a WARNING to stderr naming those stories -- even in a MIXED release where some @@ -64,6 +48,9 @@ dry-run "transitioned" is inherently just a list of candidates, not evidence anything actually happened or failed to.) +This script only ever writes a story's workflow state -- it does not post +any comment or other annotation ("just mark it as done" is the whole job). + All ids shown in this file's docstring and comments (e.g. story id 22222) are placeholders, not real Shortcut story ids. The workflow and state ids (500000005, 500000045, 500000010, 500000728) are real Shortcut workflow/ @@ -78,14 +65,6 @@ import urllib.error import urllib.request -# The story-comment POST mechanics are shared with reconcile_deploy_ready.py -# -- see shortcut_comment.py's own docstring for why. build/ci is not a -# package (see tests/conftest.py), but a plain sibling-module import works -# both when this file is run directly (python3 puts its own directory on -# sys.path[0]) and under pytest (the test conftest adds build/ci to -# sys.path the same way). -import shortcut_comment - SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" # Sefaria's "Standard" Shortcut workflow: "Deploy Ready" -> "Done". These are @@ -98,10 +77,6 @@ DEFAULT_FROM_STATE_ID = 500000045 DEFAULT_DONE_STATE_ID = 500000010 -# Used only to build a GitHub PR link in the write-back comment -- this -# script never calls `gh` or the GitHub API itself. -DEFAULT_REPO = "Sefaria/Sefaria-Project" - def die(message: str) -> None: print(f"ERROR: {message}", file=sys.stderr) @@ -179,57 +154,6 @@ def transition_stories(to_transition, done_state_id, token, max_workers=8): return transitioned, failed -def _pr_numbers_for_story(story_id, commits): - """PR numbers of every commit in this release's own `commits` list - (from --input's shipped-stories.json) that carried this story id, in - the order they appear there. A story can ship via more than one - commit/PR in the same release (e.g. a same-day fix-up), so this - collects all of them rather than just the first match. Returns [] - (not an error) when none are resolvable -- an older shipped-stories.json - without a `commits` key, or a story whose only carrying commit had no - PR number, both degrade to "no PR reference available" in the comment - text rather than raising.""" - sid = str(story_id) - numbers = [] - seen = set() - for c in commits: - pr_number = c.get("pr_number") - if pr_number and sid in (c.get("story_ids") or []) and pr_number not in seen: - seen.add(pr_number) - numbers.append(pr_number) - return numbers - - -def _release_comment_text(story, data, repo): - """Write-back comment text for a story THIS RUN actually transitioned. - Unlike reconcile_deploy_ready.py's backfill sweep, there's no release - ambiguity to resolve here: --input IS this release's own - shipped-stories.json, so `version`/`chart_version`/`release_date` are - exactly the release this story just shipped in -- state that fact - directly rather than re-deriving it from git the way the sweep has to - for stories it didn't just observe shipping in real time.""" - version = data.get("version") or "an unresolved version" - chart_version = data.get("chart_version") - release_date = data.get("release_date") - # Human-readable date only (no time-of-day/timezone noise) -- release_date - # is an ISO 8601 timestamp like "2026-08-31T07:17:36Z". - release_date_human = release_date.split("T")[0] if release_date else "an unresolved date" - - pr_numbers = _pr_numbers_for_story(story.get("id"), data.get("commits") or []) - if pr_numbers: - pr_line = "PR(s): " + ", ".join(f"https://github.com/{repo}/pull/{n}" for n in pr_numbers) - else: - pr_line = "PR(s): not available in this release's shipped-stories data." - - chart_part = f" (chart {chart_version})" if chart_version else "" - - return ( - "\U0001F916 Automated update — posted by the prod release pipeline; no reply expected.\n" - f"Shipped in prod release {version}{chart_part}, released {release_date_human}.\n" - f"{pr_line}" - ) - - def _ids(stories): return sorted((s.get("id") for s in stories), key=lambda x: (x is None, x)) @@ -265,11 +189,6 @@ def build_arg_parser(): parser.add_argument("--workflow-id", type=int, default=DEFAULT_WORKFLOW_ID) parser.add_argument("--from-state-id", type=int, default=DEFAULT_FROM_STATE_ID) parser.add_argument("--done-state-id", type=int, default=DEFAULT_DONE_STATE_ID) - parser.add_argument("--repo", default=DEFAULT_REPO, help=f"GitHub repo for the comment's PR link(s), default {DEFAULT_REPO}") - parser.add_argument( - "--no-comment", action="store_true", - help="Transition stories but skip posting the write-back release comment.", - ) parser.add_argument("--max-workers", type=int, default=8) return parser @@ -302,9 +221,6 @@ def main(): stories, args.workflow_id, args.from_state_id, args.done_state_id, ) - comment_posted = [] - comment_failed = [] - if args.dry_run: for story in to_transition: print( @@ -313,38 +229,11 @@ def main(): f"{args.from_state_id} to {args.done_state_id}", file=sys.stderr, ) - # Preview-only: never calls Shortcut. Shown so a --dry-run run - # says what it WOULD post, matching how it already says what it - # would transition. - if not args.no_comment: - preview = "\n".join(f"DRY RUN: {line}" for line in _release_comment_text(story, data, args.repo).splitlines()) - print(f"DRY RUN: would post comment on story {story.get('id')}:\n{preview}", file=sys.stderr) transitioned = _ids(to_transition) failed = [] else: transitioned, failed = transition_stories(to_transition, args.done_state_id, token, args.max_workers) - # Comment ONLY on a story THIS RUN actually transitioned -- never - # already_done/skipped (those paths never reach to_transition at - # all), and never a story that was a to_transition CANDIDATE but - # whose PUT itself failed (excluded via transitioned_ids below). - # transitions are naturally once-only (a transitioned story leaves - # Deploy Ready, so classify_stories routes it to already_done on any - # re-run) -- no separate dedupe index is needed to keep a re-run - # from re-posting. - if not args.no_comment: - transitioned_ids = set(transitioned) - for story in to_transition: - if story.get("id") not in transitioned_ids: - continue - comment_text = _release_comment_text(story, data, args.repo) - sid, ok, err = shortcut_comment.post_story_comment(story.get("id"), comment_text, token) - if ok: - comment_posted.append(sid) - else: - warn(f"Failed to post release comment on story {sid}: {err}") - comment_failed.append({"id": sid, "error": err}) - skipped_detail = _skipped_detail(skipped_other_state, skipped_different_workflow) summary = { @@ -355,8 +244,6 @@ def main(): "skipped_other_state": len(skipped_other_state), "skipped_different_workflow": len(skipped_different_workflow), "failed": len(failed), - "comment_posted": len(comment_posted), - "comment_failed": len(comment_failed), }, "transitioned": sorted(transitioned, key=lambda x: (x is None, x)), "already_done": _ids(already_done), @@ -364,8 +251,6 @@ def main(): "skipped_different_workflow": _ids(skipped_different_workflow), "skipped_detail": skipped_detail, "failed": sorted(failed, key=lambda x: (x is None, x)), - "comment_posted": sorted(comment_posted, key=lambda x: (x is None, x)), - "comment_failed": sorted(comment_failed, key=lambda d: (d["id"] is None, d["id"])), "hydrated": hydrated, "unresolved_story_ids": sorted(unresolved_story_ids), } diff --git a/build/ci/merge_release_backfill.py b/build/ci/merge_release_backfill.py new file mode 100644 index 0000000000..bd26f40b9b --- /dev/null +++ b/build/ci/merge_release_backfill.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Merge reconcile_deploy_ready.py's "shipped in THIS release" backfill stories +into shipped_stories.py's own output, for the release-notes prose step. + +Why this exists: reconcile_deploy_ready.py's org-wide sweep exists because +nothing else revisits a Deploy Ready story once it falls outside the +current release's git-range scan (RC2). Most of what it finds shipped in +EARLIER releases, and must never reach today's announcement -- that's why +its report is written to $RUNNER_TEMP and never touched by the prose step +(see that script's own docstring). But some of what it finds -- a race, a +discovery gap RC1 didn't close, a story that just never got the write-back +it deserved -- shipped in the CURRENT release, same as everything +shipped_stories.py's own git-range scan already found. Silently excluding +THOSE from the announcement is a different flavor of the same underlying +mistake ("say what actually shipped today"), just by omission instead of +leakage. This script is the one, single, deterministic place that decides +which backfilled stories cross that line -- never the prose agent itself, +never a prompt instruction. + +The decision was ALREADY MADE upstream, in reconcile_deploy_ready.py's own +classify_candidates: a shipped entry there carries `hydrated_story` if and +only if its `shipping_release_tag` (the TRUE release, resolved via +resolve_shipping_release_tag's `git tag --contains` lookup) equals the +CURRENT prod tag reconcile_deploy_ready.py was run against. This script +trusts that signal completely and does nothing else -- it is a pure, +mechanical merge, not a second opinion. `hydrated_story` is already in +shipped_stories.py's own hydrated-story shape (id, name, description, url, +workflow_id, workflow_state_id, story_type), built by +reconcile_deploy_ready.py from data it already had in memory -- no extra +Shortcut API call here either. + +Reads --shipped-stories-out (shipped_stories.py's own --out file) and +--reconcile-report (reconcile_deploy_ready.py's own --out file), and writes +the merged result to --out (default: overwrite --shipped-stories-out in +place, so the prose step's existing "read shipped-stories.json" step needs +no change at all). Deduplicates by story id against what shipped_stories.py +ALREADY found -- a story RC1's text/PR-link discovery independently found +in this same range must not be double-counted or duplicated in the merged +`stories` list. + +CRITICAL: this script only ever reads reconcile_deploy_ready.py's report to +extract the pre-filtered, pre-decided `hydrated_story` entries. It never +reads (or writes back) `shipped`/`pending`/`triage` wholesale, and it never +makes its own judgment about which release a story belongs to -- that +judgment already happened, once, in reconcile_deploy_ready.py, using git +evidence. Running this script does not, by itself, change what's readable +by the release-notes prose step: it only ever touches shipped-stories.json, +which was already that step's designated input before this script existed. +The reconcile report itself must still never be passed to, or made +readable by, that step -- this script's whole job is to extract the one +safe, pre-filtered slice from it and leave the rest behind in $RUNNER_TEMP. + +Usage: + python3 merge_release_backfill.py --shipped-stories-out shipped-stories.json \ + --reconcile-report reconcile-deploy-ready-report.json [--out shipped-stories.json] + +Stdlib only -- no third-party dependencies, matching the rest of this +pipeline's scripts. + +All story ids in this file's docstring and comments (e.g. 11111) are +placeholders, not real Shortcut story ids. +""" + +import argparse +import json +import sys + + +def die(message: str) -> None: + print(f"ERROR: {message}", file=sys.stderr) + sys.exit(1) + + +def warn(message: str) -> None: + print(f"WARNING: {message}", file=sys.stderr) + + +def backfill_stories_from_report(report): + """Every `hydrated_story` already attached to report['shipped'] -- + i.e. every story reconcile_deploy_ready.py ALREADY determined belongs + in the CURRENT release (shipping_release_tag == the prod tag that run + was against). This function makes no decision of its own: a shipped + entry with no `hydrated_story` (an earlier or unresolvable release) is + silently skipped, exactly as it should be -- fail closed, never guess + a story into an announcement this script didn't independently verify + and has no way to.""" + return [s["hydrated_story"] for s in (report.get("shipped") or []) if s.get("hydrated_story")] + + +def merge(shipped_stories_data, backfill_stories): + """Fold `backfill_stories` (already in shipped_stories.py's own + hydrated-story shape) into `shipped_stories_data`'s `story_ids` / + `stories`, deduplicated by story id. A story shipped_stories.py's own + git-range + RC1 discovery ALREADY found independently must not be + duplicated -- existing entries always win the dedupe (this script + never overwrites data shipped_stories.py already hydrated for itself). + + Returns (merged_data, added_ids) so the caller can report exactly what + was added, for visibility -- mirrors shipped_stories.py's own + stories_from_shortcut_pr_link provenance field.""" + existing_ids = set(str(sid) for sid in (shipped_stories_data.get("story_ids") or [])) + stories = list(shipped_stories_data.get("stories") or []) + added_ids = [] + + for story in backfill_stories: + sid = str(story.get("id")) + if sid in existing_ids: + continue + existing_ids.add(sid) + added_ids.append(sid) + stories.append(story) + + shipped_stories_data["story_ids"] = sorted(existing_ids, key=int) + shipped_stories_data["stories"] = stories + # Provenance: which ids were folded in by the reconciliation sweep, + # distinct from shipped_stories.py's own git-range/RC1 discovery -- + # same purpose as that script's own stories_from_shortcut_pr_link + # field (say what only the sweep knew). + shipped_stories_data["stories_from_reconciliation_backfill"] = sorted(added_ids, key=int) + + return shipped_stories_data, added_ids + + +def build_arg_parser(): + parser = argparse.ArgumentParser( + description="Merge reconcile_deploy_ready.py's current-release backfill stories " + "into shipped_stories.py's own output, before the release-notes prose step runs.", + ) + parser.add_argument("--shipped-stories-out", required=True, help="Path to shipped_stories.py's own --out file") + parser.add_argument("--reconcile-report", required=True, help="Path to reconcile_deploy_ready.py's own --out file") + parser.add_argument( + "--out", + help="Where to write the merged result (default: overwrite --shipped-stories-out in place)", + ) + return parser + + +def main(): + args = build_arg_parser().parse_args() + + try: + with open(args.shipped_stories_out, encoding="utf-8") as f: + shipped_stories_data = json.load(f) + except (OSError, json.JSONDecodeError) as e: + die(f"Could not read/parse --shipped-stories-out {args.shipped_stories_out!r}: {e}") + + try: + with open(args.reconcile_report, encoding="utf-8") as f: + report = json.load(f) + except (OSError, json.JSONDecodeError) as e: + die(f"Could not read/parse --reconcile-report {args.reconcile_report!r}: {e}") + + backfill_stories = backfill_stories_from_report(report) + merged, added_ids = merge(shipped_stories_data, backfill_stories) + + out_path = args.out or args.shipped_stories_out + with open(out_path, "w", encoding="utf-8") as f: + json.dump(merged, f, indent=2, ensure_ascii=False) + f.write("\n") + + if added_ids: + print(f"Merged {len(added_ids)} backfilled stor{'y' if len(added_ids) == 1 else 'ies'} " + f"that shipped in the current release: {added_ids}") + else: + print("No backfilled stories belong to the current release -- nothing to merge.") + + +if __name__ == "__main__": + main() diff --git a/build/ci/reconcile_deploy_ready.py b/build/ci/reconcile_deploy_ready.py index 1fe3cefb5b..622ef04ad4 100644 --- a/build/ci/reconcile_deploy_ready.py +++ b/build/ci/reconcile_deploy_ready.py @@ -30,9 +30,9 @@ alone and reported for a human to look at -- this is the part of the output that actually needs eyes on it. -Four guards apply before a linked PR counts as evidence a story shipped -- -each one caught a real false positive while this script was being built -against live data: +Five guards apply before a linked PR counts as evidence a story shipped -- +each one caught a real false positive while this script was being built (or +maintained) against live data: 1. repository_id must be Sefaria-Project's (500000103). A story can link a PR from a DIFFERENT repo (e.g. a docs or infra repo); resolving that @@ -45,9 +45,20 @@ (preprod -> prod, or master -> preprod) instead of, or alongside, the actual feature PR. A promotion PR merges constantly and proves nothing about whether THIS story's own change reached prod. - 3. merged must be true. An open or closed-without-merging PR is not + 3. The PR's own HEAD (source) branch must NOT be a long-lived environment + branch (master, preprod, prod). Verified live: a promotion PR merging + preprod INTO master passes guards 1-2 and 4 cleanly -- it's merged, + against the right repo, and its target really is "master" -- yet it's + still a promotion merge, not a feature PR. Guard 2 alone cannot catch + this shape (a promotion merge legitimately targets master); only the + HEAD branch gives it away. This is the exact live false positive that + was found and fixed: a story was classified `shipped` on the strength + of a promotion PR shaped exactly this way, while its genuine feature + PR (correctly) failed guard 2 for targeting a hotfix branch instead of + master directly. + 4. merged must be true. An open or closed-without-merging PR is not evidence of anything having shipped. - 4. workflow_id must be Sefaria's "Standard" workflow (500000005), and + 5. workflow_id must be Sefaria's "Standard" workflow (500000005), and workflow_state_id must be exactly the numeric Deploy Ready state id (500000045) within it. The Shortcut state named "Deploy Ready" (note: the real name carries a trailing space, "Deploy Ready ") is @@ -58,6 +69,11 @@ triage with its actual workflow/state ids reported instead, mirroring mark_stories_deployed.py's skipped_different_workflow handling. +Guards 1-4 are PR-level and live in shortcut_pr_guards.py, shared with +shipped_stories.py's RC1 fallback -- see that module's own docstring. Guard +5 is story-level and specific to this sweep; it stays local, in +classify_stories below. + Enumeration uses the token'd search endpoint (`search/stories?query=state:"Deploy Ready" !is:archived`), paginated via its `next` cursor -- NOT `iterations-get-active`, which is silently scoped @@ -84,38 +100,38 @@ Emits a JSON report (--out) with all three buckets in full, plus a readable stdout summary. CRITICAL: this script never reads or writes -shipped-stories.json and never feeds the release-notes prose step -- the -stories it backfills shipped in EARLIER releases, and leaking them into -today's release announcement would have Slack claim a dozen old features -shipped today. Reconciliation transitions Shortcut state only; it has no -opinion about what today's release notes should say. - -Immediately after a story is ACTUALLY transitioned by this run, a short -write-back comment is posted on it via `POST /stories/{id}/comments` -(shortcut_comment.py, shared with mark_stories_deployed.py's own -write-back) -- but the SAME "old features shipped today" mistake this -script's whole shipped-stories.json separation exists to avoid can just as -easily happen inside a single Shortcut comment, so the comment text is -built differently here than in mark_stories_deployed.py. That script knows -the current release's own version/chart/date directly (it's reading that -release's own shipped-stories.json); this script does NOT -- a story it -backfills shipped in some EARLIER release, and naming the CURRENT prod tag -in its comment would tell a reader it shipped TODAY, which is false. So -this script instead asks git for the TRUE release: `git tag --list +shipped-stories.json itself and never talks to the release-notes prose +step directly -- the stories it backfills mostly shipped in EARLIER +releases, and leaking them into today's release announcement would have +Slack claim a dozen old features shipped today. This script only ever +transitions Shortcut state; it never posts a comment or any other +annotation ("just mark it as done" is the whole job here). + +One exception exists, and it is handled by a SEPARATE, later, explicitly +filtered step -- never by this script writing to shipped-stories.json +itself: a story this sweep backfills can, in the ordinary case, ALSO have +shipped in the CURRENT release rather than an earlier one (the sweep exists +because nothing else revisits Deploy Ready stories, including ones from +THIS release that a race or a discovery gap missed). That case belongs in +today's announcement -- silently excluding it would just be a different +flavor of the same failure this whole separation defends against, this +time by omission instead of leakage. So every shipped entry in this +report also carries `shipping_release_tag`: the TRUE release that shipped +it, resolved via `resolve_shipping_release_tag()` (`git tag --list 'prod/*' --contains --sort=creatordate | head -1` -- the -first (earliest-created) prod/* tag that actually contains the winning -PR's merge commit, i.e. the release that really carried it. If that can't -be resolved for any reason (shallow checkout, tag history gap, ...), the -comment degrades HONESTLY: it says only that the story was detected as -already present in production as of the current prod tag, and names the -PR -- it never guesses or implies a specific release. - -A comment is posted ONLY for a story this run actually transitioned -- -never for pending/triage, and never merely because a dry-run run -classified it as shipped. A comment failure is logged and recorded in -`comment_failed` but never fails the run or rolls back the transition. -Posting is skipped entirely in dry-run (which instead reports what WOULD -be posted) and with --no-comment. +first, earliest-created, prod/* tag that actually contains the winning +PR's merge commit). When `shipping_release_tag` equals the CURRENT +`prod_tag`, the entry ALSO carries `hydrated_story` -- a story object in +shipped_stories.py's own hydrated-story shape, built from data this sweep +already fetched (no extra API call) -- so a separate, deterministic +downstream script (`merge_release_backfill.py`) can fold exactly that +story, and only that story, into shipped-stories.json before the prose +step runs. Every other shipped entry, and the reconcile report as a +whole, is never read by that step or any step after it. `resolve_ +shipping_release_tag()` returns None when it can't determine a release at +all (shallow checkout, tag history gap, ...); that case is EXCLUDED from +`hydrated_story` too -- fail closed, never guess a story into an +announcement. Usage: python3 reconcile_deploy_ready.py [--dry-run] @@ -154,13 +170,6 @@ # conftest adds build/ci to sys.path the same way). import shortcut_pr_guards -# The story-comment POST mechanics are shared with mark_stories_deployed.py -# -- see shortcut_comment.py's own docstring for why (and for why the -# comment TEXT itself is deliberately NOT shared -- this script and -# mark_stories_deployed.py know different things about which release -# actually shipped a story). -import shortcut_comment - SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" SHORTCUT_API_ROOT = "https://api.app.shortcut.com" @@ -315,21 +324,20 @@ def is_ancestor_of_prod(oid, prod_tag): def resolve_shipping_release_tag(oid): """The TRUE release that shipped commit `oid`: the first (earliest - created) `prod/*` tag that actually contains it. Used ONLY for the - write-back comment's text -- see the module docstring for why this - script cannot just name the current `--prod-tag` the way - mark_stories_deployed.py names its own release: a story backfilled by - this sweep almost never shipped in the CURRENT release, and a comment - claiming otherwise would be actively misleading, not just imprecise. + created) `prod/*` tag that actually contains it. Used to decide + whether a backfilled story belongs in TODAY's release notes -- see the + module docstring's `shipping_release_tag` / `hydrated_story` + paragraph and `classify_candidates` below. `--sort=creatordate` (ascending, oldest first) is deliberate and the OPPOSITE of resolve_default_prod_tag's `-creatordate` above -- that one wants the newest tag (today's release); this one wants the OLDEST tag that still contains the commit, i.e. the first release it ever reached. Returns None if the lookup fails outright (non-fatal -- - logged and the caller degrades the comment text honestly) or if no + logged, and the caller must fail closed: never guess or default to + "belongs to the current release" when this is unresolvable) or if no prod/* tag contains it at all (e.g. a shallow checkout, or a genuine - gap in tag history) -- both cases must never be guessed past.""" + gap in tag history).""" proc = subprocess.run( ["git", "tag", "--list", "prod/*", "--contains", oid, "--sort=creatordate"], capture_output=True, @@ -342,46 +350,6 @@ def resolve_shipping_release_tag(oid): return tags[0] if tags else None -def _reconcile_comment_text(story_entry, oid_by_pr, prod_tag, repo): - """Write-back comment text for a story THIS SWEEP actually transitioned. - See resolve_shipping_release_tag and the module docstring for why this - resolves the TRUE shipping release from git rather than naming the - current --prod-tag: naming the current release for a backfilled story - would be exactly the "old features shipped today" mistake this whole - script's two-output separation (never touching shipped-stories.json) - exists to prevent -- just showing up in a Shortcut comment instead of a - Slack post.""" - pr_numbers = story_entry.get("shipped_via_prs") or [] - if pr_numbers: - pr_line = "PR(s): " + ", ".join(f"https://github.com/{repo}/pull/{n}" for n in pr_numbers) - else: - pr_line = "PR(s): unavailable." - - shipping_tag = None - if pr_numbers: - oid = oid_by_pr.get(pr_numbers[0]) - if oid: - shipping_tag = resolve_shipping_release_tag(oid) - - if shipping_tag: - headline = ( - f"Detected as shipped in {shipping_tag} — found while reconciling the Deploy " - "Ready backlog (not part of today's release)." - ) - else: - headline = ( - f"Detected as already present in production as of {prod_tag} — found while " - "reconciling the Deploy Ready backlog; the exact shipping release could not be " - "determined." - ) - - return ( - "\U0001F916 Automated update — posted by the Deploy Ready reconciliation sweep; no reply expected.\n" - f"{headline}\n" - f"{pr_line}" - ) - - def transition_story(story_id, done_state_id, token): """PUT a workflow_state_id update to Shortcut. Mirrors mark_stories_deployed.transition_story exactly (kept as its own copy @@ -438,7 +406,7 @@ def _triage_context(story): def _diagnose_linked_pr(pr, repo_id, target_branch): - """Which of the three PR-level guards (see qualifying_prs / + """Which of the four PR-level guards (see qualifying_prs / shortcut_pr_guards.passes_pr_guards) this specific linked PR fails, if any. Diagnostic ONLY -- classification itself never reads this; it exists purely so a triage story's report entry can say WHY a linked PR @@ -450,6 +418,8 @@ def _diagnose_linked_pr(pr, repo_id, target_branch): failed.append(f"wrong repo (repository_id={pr.get('repository_id')}, expected {repo_id})") if pr.get("target_branch_name") != target_branch: failed.append(f"wrong target branch ({pr.get('target_branch_name')!r}, expected {target_branch!r})") + if pr.get("branch_name") in shortcut_pr_guards.LONG_LIVED_ENV_BRANCHES: + failed.append(f"promotion PR (head branch {pr.get('branch_name')!r} is a long-lived environment branch)") return failed @@ -509,13 +479,48 @@ def classify_stories(stories, repo_id, target_branch): return triage, candidates +def _story_for_release_notes(story): + """Build a story entry in EXACTLY shipped_stories.py's own + hydrated-story shape (id, name, description, url, workflow_id, + workflow_state_id, story_type -- see that script's fetch_story) + directly from the story object this sweep already fetched via + search/stories -- its "detail=full" response already carries every one + of these fields, so no extra Shortcut API call is needed here. Used + ONLY for a shipped story whose `shipping_release_tag` equals the + CURRENT prod tag -- see classify_candidates and the module docstring.""" + return { + "id": story.get("id"), + "name": story.get("name"), + "description": story.get("description"), + "url": story.get("app_url"), + "workflow_id": story.get("workflow_id"), + "workflow_state_id": story.get("workflow_state_id"), + "story_type": story.get("story_type"), + } + + def classify_candidates(candidates, oid_by_pr, prod_tag): """For each (story, qualifying_prs) candidate, check every qualifying PR's merge commit for prod ancestry and split into shipped / pending. A PR whose merge oid never resolved, or whose ancestry check itself couldn't run, counts as inconclusive -- never as "in prod" -- so a resolution failure can only ever push a story toward pending (leave it - alone), never wrongly toward shipped (a mutation).""" + alone), never wrongly toward shipped (a mutation). + + Every shipped entry also carries `shipping_release_tag` -- the TRUE + release that shipped it (resolve_shipping_release_tag, from the first + confirmed-in-prod qualifying PR's merge commit) -- computed + unconditionally, independent of --apply/--dry-run: this is a fact + about git history, not about whether THIS run happens to mutate + Shortcut, so a dry-run report is exactly as trustworthy an input to + the downstream release-notes merge decision as a live one. ONLY when + `shipping_release_tag` equals the CURRENT `prod_tag` -- i.e. this + story provably shipped in TODAY's release, not an earlier one -- the + entry ALSO carries `hydrated_story` (see _story_for_release_notes). + That field's presence is the ONLY signal merge_release_backfill.py + uses to decide whether to fold a backfilled story into + shipped-stories.json; every other shipped entry (a different or + unresolvable release) carries no such field and is never merged.""" shipped = [] pending = [] for story, prs in candidates: @@ -530,7 +535,12 @@ def classify_candidates(candidates, oid_by_pr, prod_tag): entry = _story_summary(story) entry["qualifying_prs"] = sorted(pr["number"] for pr in prs) if shipped_via: - entry["shipped_via_prs"] = sorted(shipped_via) + shipped_via_sorted = sorted(shipped_via) + entry["shipped_via_prs"] = shipped_via_sorted + shipping_tag = resolve_shipping_release_tag(oid_by_pr[shipped_via_sorted[0]]) + entry["shipping_release_tag"] = shipping_tag + if shipping_tag == prod_tag: + entry["hydrated_story"] = _story_for_release_notes(story) shipped.append(entry) else: pending.append(entry) @@ -558,14 +568,19 @@ def print_summary(report): status = "transitioned" else: status = f"FAILED: {s.get('transition_error')}" - print(f" {s['id']} {s.get('name', '')!r} via PR(s) {s.get('shipped_via_prs')} [{status}]") - if s.get("would_comment"): - preview = "\n".join(f" {line}" for line in s["would_comment"].splitlines()) - print(f" would post comment:\n{preview}") - elif s.get("comment_posted"): - print(" comment posted") - elif s.get("comment_error"): - print(f" comment FAILED: {s['comment_error']}") + # belongs-to-current-release marker: hydrated_story's presence is + # the same signal merge_release_backfill.py acts on downstream -- + # printed here too so a human reading this summary can see which + # shipped stories will (or won't) show up in today's announcement. + release_note = ( + "belongs in THIS release's notes" if s.get("hydrated_story") + else f"shipped in {s.get('shipping_release_tag')}" if s.get("shipping_release_tag") + else "shipping release could not be determined" + ) + print( + f" {s['id']} {s.get('name', '')!r} via PR(s) {s.get('shipped_via_prs')} " + f"[{status}] ({release_note})" + ) print(f"--- pending ({len(report['pending'])}) ---") for s in report["pending"]: @@ -611,10 +626,6 @@ def build_arg_parser(): f"default {DEFAULT_TARGET_BRANCH!r}", ) parser.add_argument("--out", help="Write the machine-readable JSON report to this path") - parser.add_argument( - "--no-comment", action="store_true", - help="Transition stories but skip posting the write-back release comment.", - ) parser.add_argument("--max-workers", type=int, default=8) return parser @@ -652,8 +663,6 @@ def main(): apply_mutations = args.apply and not args.dry_run failed_transitions = [] - comment_posted = [] - comment_failed = [] if apply_mutations: transitioned_ids, failed_transitions = transition_stories( [s["id"] for s in shipped], DONE_STATE_ID, token, args.max_workers @@ -664,33 +673,9 @@ def main(): s["transitioned"] = s["id"] in transitioned_set if s["id"] in failed_by_id: s["transition_error"] = failed_by_id[s["id"]] - - # Comment ONLY on a story THIS RUN actually transitioned -- never a - # candidate whose PUT itself failed, and never pending/triage. - # Re-runs never re-post: a transitioned story leaves Deploy Ready, - # so it's simply absent from the NEXT run's search results -- no - # separate dedupe index is needed. - if not args.no_comment: - for s in shipped: - if not s["transitioned"]: - continue - comment_text = _reconcile_comment_text(s, oid_by_pr, prod_tag, args.repo) - sid, ok, err = shortcut_comment.post_story_comment(s["id"], comment_text, token) - if ok: - comment_posted.append(sid) - s["comment_posted"] = True - else: - warn(f"Failed to post release comment on story {sid}: {err}") - comment_failed.append({"id": sid, "error": err}) - s["comment_error"] = err else: for s in shipped: s["transitioned"] = None # not attempted -- dry-run - # Preview-only: never calls Shortcut, but DOES shell out to git - # (resolve_shipping_release_tag) -- read-only local introspection, - # not a live mutation, so it's safe to compute even in dry-run. - if not args.no_comment: - s["would_comment"] = _reconcile_comment_text(s, oid_by_pr, prod_tag, args.repo) report = { "prod_tag": prod_tag, @@ -700,14 +685,10 @@ def main(): "shipped": len(shipped), "pending": len(pending), "triage": len(triage), - "comment_posted": len(comment_posted), - "comment_failed": len(comment_failed), }, "shipped": shipped, "pending": pending, "triage": triage, - "comment_posted": sorted(comment_posted), - "comment_failed": sorted(comment_failed, key=lambda d: (d["id"] is None, d["id"])), } print_summary(report) diff --git a/build/ci/shipped_stories.py b/build/ci/shipped_stories.py index 6d315df74e..6458825ec6 100755 --- a/build/ci/shipped_stories.py +++ b/build/ci/shipped_stories.py @@ -18,14 +18,15 @@ nothing about whether that story's own change shipped (verified live: PRs whose head branch was `preprod` or `master` each resolved to a real story this way). The single search result is also re-checked against the SAME -three PR-level guards `reconcile_deploy_ready.py`'s org-wide sweep applies -(merged / Sefaria-Project repo / target branch master; shared via -shortcut_pr_guards.py so the two scripts cannot silently drift apart) before -its id is adopted -- a match that fails those guards is a warn-and-skip, not -a fallback of last resort. The id is adopted ONLY when the search resolves -to EXACTLY one story AND that story's PR passes those guards; ids recovered -this way are also surfaced separately in `stories_from_shortcut_pr_link` so -a report can say what only Shortcut knew. +four PR-level guards `reconcile_deploy_ready.py`'s org-wide sweep applies +(merged / Sefaria-Project repo / target branch master / head branch not a +long-lived environment branch; shared via shortcut_pr_guards.py so the two +scripts cannot silently drift apart) before its id is adopted -- a match +that fails those guards is a warn-and-skip, not a fallback of last resort. +The id is adopted ONLY when the search resolves to EXACTLY one story AND +that story's PR passes those guards; ids recovered this way are also +surfaced separately in `stories_from_shortcut_pr_link` so a report can say +what only Shortcut knew. Revert commits (`Revert "..."`, `Revert: ...`, `revert(...)`) never contribute story ids to the shipped set; their suppressed ids are surfaced separately in `reverted_commits` instead of being silently dropped. @@ -88,7 +89,14 @@ # `branch` value already resolved via fetch_pr_branch for story-id # extraction), not its target branch -- shortcut_pr_guards' target-branch # guard covers that side separately. -LONG_LIVED_ENV_BRANCHES = frozenset({"master", "preprod", "prod"}) +# +# Re-exported from shortcut_pr_guards (guard #4 there) rather than a second +# local copy -- these two scripts already drifted apart once on whether a +# head-branch guard existed at all (reconcile_deploy_ready.py had none until +# a promotion PR slipped a story into "shipped" live); naming the same +# frozenset object in both places is what actually prevents a second drift, +# not just matching the values by hand. +LONG_LIVED_ENV_BRANCHES = shortcut_pr_guards.LONG_LIVED_ENV_BRANCHES # Shortcut (SC) story id patterns recognized in a commit subject or a PR # branch name. Kept intentionally short: `\bsc[-_](\d+)\b` (pattern 1) has a @@ -367,9 +375,10 @@ def fetch_story_by_pr_link(pr_number, token): Adopts the id ONLY when the search returns EXACTLY one story AND that story's OWN linked-PR entry for this exact PR number passes the same - three PR-level guards reconcile_deploy_ready.py's sweep applies (merged - / Sefaria-Project repo / target branch master -- see - shortcut_pr_guards.py). That second check matters because a bare + four PR-level guards reconcile_deploy_ready.py's sweep applies (merged + / Sefaria-Project repo / target branch master / head branch not a + long-lived environment branch -- see shortcut_pr_guards.py). That + second check matters because a bare `pr:` match only proves Shortcut linked SOME story to this PR number -- not that this PR is real shipping evidence for it. A promotion PR (head branch `master`/`preprod`/`prod`) resolves via this @@ -422,8 +431,8 @@ def fetch_story_by_pr_link(pr_number, token): warn( f"Shortcut PR-link lookup for PR #{pr_number} resolved to story {story_id}, but " "that PR does not pass the shipping-evidence guards (merged / Sefaria-Project " - "repo / target branch master) -- e.g. a promotion or branch-sync merge rather " - "than the real feature PR. Skipping." + "repo / target branch master / head branch not long-lived) -- e.g. a promotion " + "or branch-sync merge rather than the real feature PR. Skipping." ) return pr_number, None diff --git a/build/ci/shortcut_comment.py b/build/ci/shortcut_comment.py deleted file mode 100644 index 87663b80b0..0000000000 --- a/build/ci/shortcut_comment.py +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env python3 -""" -Shared "post a comment on a Shortcut story" helper. - -Today this pipeline only ever WRITES a story's workflow state (a PUT that -changes `workflow_state_id`) -- nothing records WHICH release actually -carried a story, so a person reading the story inside Shortcut can see it -became Done but has no way to tell what shipped it without going and -digging through CI logs or git. Both `mark_stories_deployed.py` (this -release's own shipped stories) and `reconcile_deploy_ready.py` (older -stories backfilled by the org-wide sweep) close that gap by posting a -short, factual write-back comment immediately after a successful -transition -- but they need the exact same POST mechanics and the exact -same non-fatal error handling around it, so that lives here once. Two -copies of this would be exactly the drift failure the shared -shortcut_pr_guards.py module already exists to prevent for the PR-level -guards -- same reasoning, same fix. - -What differs between the two callers is not HOW to post a comment, but -WHAT the comment says: `mark_stories_deployed.py` knows the current -release's own version/chart/date (it's reading that release's own -shipped-stories.json) and can just say so. `reconcile_deploy_ready.py` -does NOT know that -- a story it backfills shipped in some EARLIER -release, and if its comment named the CURRENT prod tag, a reader would -reasonably conclude that story shipped in TODAY's release. That's exactly -the "old features shipped today" error class the shipped-stories.json / -reconcile-report separation in this codebase's other docstrings exists to -avoid, just showing up in a different place (a Shortcut comment instead of -a Slack post). So comment-TEXT construction stays in each caller, where -the release-identity knowledge already lives; only the POST mechanics are -shared here. - -Stdlib only -- no third-party dependencies, matching both callers' -dependency posture. -""" - -import json -import urllib.error -import urllib.request - -SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" - - -def post_story_comment(story_id, text, token): - """POST a comment onto a story (`POST /stories/{id}/comments`). - Returns (story_id, ok, error). Mirrors the transition_story functions - in both callers: a failure here must never raise, and must never be - conflated with a failed transition -- by the time this is ever called, - the state change has ALREADY succeeded. The comment is a best-effort - annotation on top of a real, already-durable state change, not a - precondition for it -- so a comment failure is reported (the caller - warns and tracks it) but never rolls anything back and never fails the - run on its own.""" - url = f"{SHORTCUT_API_BASE}/stories/{story_id}/comments" - body = json.dumps({"text": text}).encode("utf-8") - req = urllib.request.Request(url, data=body, method="POST") - req.add_header("Shortcut-Token", token) - req.add_header("Content-Type", "application/json") - try: - with urllib.request.urlopen(req, timeout=20) as resp: - resp.read() - return story_id, True, None - except urllib.error.HTTPError as e: - return story_id, False, f"HTTP {e.code} {e.reason}" - except Exception as e: # noqa: BLE001 - a comment failure must never abort the run or roll back the transition - return story_id, False, str(e) diff --git a/build/ci/shortcut_pr_guards.py b/build/ci/shortcut_pr_guards.py index 89c67159e4..5e27e771c5 100644 --- a/build/ci/shortcut_pr_guards.py +++ b/build/ci/shortcut_pr_guards.py @@ -6,13 +6,21 @@ `reconcile_deploy_ready.py` (RC2's org-wide Deploy Ready sweep) need to answer the exact same question about a PR that Shortcut says is linked to a story: does this PR actually prove that story's change reached prod? A -linked PR is NOT automatically that evidence -- three guards apply, and -both scripts must apply the SAME three guards or they will silently drift -apart (verified live: a promotion PR, e.g. head branch `preprod` merging -into `master`, or `master` merging into `preprod`, resolves via +linked PR is NOT automatically that evidence -- FOUR guards apply, and +both scripts must apply the SAME four guards or they will silently drift +apart. They already have, twice, in opposite directions: RC1's PR-link +fallback initially had no guards at all (a promotion PR resolved via `search/stories?query=pr:` to a real story just as readily as that -story's actual feature PR does -- but proves nothing about whether that -story's own change shipped). +story's actual feature PR); then, after guards 1-3 were added here, a +promotion PR merging INTO master (rather than out of it) turned out to +still pass all three -- verified live: a real story was classified +`shipped` on the strength of a PR whose head branch was `preprod` and +target branch was `master`, a promotion merge, while its genuine feature +PR (head a hotfix/bugfix branch, target a hotfix branch) was correctly +rejected by guard 3 for not targeting `master` directly. Guard 3 alone +cannot catch this shape: a promotion merge legitimately targets `master`, +so the target-branch check has nothing to object to. The giveaway is the +SOURCE (head) branch, not the target -- hence guard 4. 1. `merged` must be true. An open or closed-without-merging PR is not evidence anything shipped. @@ -24,6 +32,13 @@ prod, or master -> preprod) merges constantly and proves nothing about whether a given story's own change reached prod -- it must never be treated as interchangeable with the real feature PR. + 4. `branch_name` (the PR's HEAD/source branch) must NOT be a long-lived + environment branch (master, preprod, prod). A promotion PR that + merges ONE of those branches INTO master (e.g. preprod -> master, the + opposite direction from guard 3's preprod/prod targets) passes guards + 1-3 cleanly -- it's merged, against the right repo, and its target + really is "master". Only the head branch reveals it's a promotion + merge, not a feature PR. This module holds the shared implementation so there is exactly one place these guards live; single-source-of-truth, not two parallel copies that a @@ -43,17 +58,27 @@ DEFAULT_TARGET_BRANCH = "master" +# Guard #4. Shared with shipped_stories.py's own pre-filter (it also skips +# the RC1 fallback lookup entirely for a commit whose PR head branch is one +# of these -- see that script's LONG_LIVED_ENV_BRANCHES, which now imports +# this same set rather than keeping a second copy) -- the two must name the +# exact same branches or they can drift apart on what counts as "long-lived" +# the same way they already drifted on whether this guard existed at all. +LONG_LIVED_ENV_BRANCHES = frozenset({"master", "preprod", "prod"}) + def passes_pr_guards(pr, repo_id=SEFARIA_PROJECT_REPO_ID, target_branch=DEFAULT_TARGET_BRANCH): """True if a single linked-PR object (a Shortcut `pull-request` entity, as found in a story's `pull_requests` or `branches[*].pull_requests`) counts as evidence that a story's change reached prod: merged, against - the right repo, targeting the right branch. See the module docstring - for why each of the three checks exists.""" + the right repo, targeting the right branch, and NOT itself a promotion + merge (head branch not long-lived). See the module docstring for why + each of the four checks exists.""" return ( pr.get("merged") is True and pr.get("repository_id") == repo_id and pr.get("target_branch_name") == target_branch + and pr.get("branch_name") not in LONG_LIVED_ENV_BRANCHES ) diff --git a/build/ci/tests/test_mark_stories_deployed.py b/build/ci/tests/test_mark_stories_deployed.py index 883af03fb0..3704683dda 100644 --- a/build/ci/tests/test_mark_stories_deployed.py +++ b/build/ci/tests/test_mark_stories_deployed.py @@ -135,8 +135,6 @@ def _boom(*args, **kwargs): "skipped_other_state": 1, "skipped_different_workflow": 1, "failed": 0, - "comment_posted": 0, - "comment_failed": 0, } @@ -172,9 +170,6 @@ def _boom(*args, **kwargs): # --- Live run (mocked urllib): transitions only the Deploy Ready bucket - def test_live_run_transitions_only_deploy_ready_stories(monkeypatch, tmp_path): - """--no-comment here to keep this test scoped to transition mechanics - only -- the write-back comment behavior has its own dedicated tests - below.""" monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") calls = [] @@ -201,7 +196,7 @@ def _fake_urlopen(req, timeout=None): monkeypatch.setattr( "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), "--no-comment", + ["mark_stories_deployed.py", "--input", str(input_path), "--workflow-id", str(WORKFLOW_ID), "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], ) @@ -240,7 +235,7 @@ def read(self): monkeypatch.setattr( "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), "--no-comment", + ["mark_stories_deployed.py", "--input", str(input_path), "--workflow-id", str(WORKFLOW_ID), "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], ) @@ -479,209 +474,3 @@ def _boom(*args, **kwargs): err = capsys.readouterr().err assert "silent no-op" not in err - -# --- write-back release comment on a successful transition -------------- -# (POST /stories/{id}/comments -- shared shortcut_comment.post_story_comment) - -RELEASE_INPUT_WITH_COMMITS = { - "version": "6.111.0-prod.2", - "chart_version": "0.87.5-prod.1", - "release_date": "2026-08-31T07:17:36Z", - "commits": [ - {"subject": "fix: a change (#3644)", "pr_number": "3644", "story_ids": ["11111"]}, - ], - "stories": [STORY_DEPLOY_READY_1], -} - - -class _OKResponse: - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def read(self): - return b"{}" - - -def test_comment_posted_on_successful_transition(monkeypatch, tmp_path): - """A story this run actually transitioned gets a write-back comment - naming THIS release's own version/chart/date and the PR that carried - it -- available directly from --input, no ambiguity to resolve (unlike - reconcile_deploy_ready.py's backfill sweep).""" - monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") - - calls = [] - - def _fake_urlopen(req, timeout=None): - calls.append((req.get_method(), req.full_url, req.data)) - return _OKResponse() - - monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) - - input_path = tmp_path / "shipped-stories.json" - input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") - - monkeypatch.setattr( - "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), - "--workflow-id", str(WORKFLOW_ID), - "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], - ) - msd.main() - - put_calls = [c for c in calls if c[0] == "PUT"] - post_calls = [c for c in calls if c[0] == "POST"] - assert put_calls == [("PUT", f"{msd.SHORTCUT_API_BASE}/stories/11111", put_calls[0][2])] - assert len(post_calls) == 1 - _, url, body = post_calls[0] - assert url == f"{msd.SHORTCUT_API_BASE}/stories/11111/comments" - text = json.loads(body.decode("utf-8"))["text"] - assert "6.111.0-prod.2" in text - assert "0.87.5-prod.1" in text - assert "2026-08-31" in text - assert "https://github.com/Sefaria/Sefaria-Project/pull/3644" in text - - -def test_comment_not_posted_on_failed_transition(monkeypatch, tmp_path): - """A story whose PUT itself failed must get NO comment -- the write-back - only follows an ACTUAL transition, never a mere candidate.""" - monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") - - import urllib.error - - def _fake_urlopen(req, timeout=None): - if req.get_method() == "PUT": - raise urllib.error.HTTPError(req.full_url, 500, "Internal Server Error", None, None) - raise AssertionError("no comment POST must be attempted for a story whose transition failed") - - monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) - - input_path = tmp_path / "shipped-stories.json" - input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") - - monkeypatch.setattr( - "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), - "--workflow-id", str(WORKFLOW_ID), - "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], - ) - msd.main() # must not raise despite the AssertionError path being unreachable - - -def test_comment_not_posted_in_dry_run(monkeypatch, tmp_path, capsys): - """--dry-run must post nothing -- but the report says what it WOULD - post, including the release identity and PR link.""" - monkeypatch.delenv("SHORTCUT_API_TOKEN", raising=False) - - def _boom(*args, **kwargs): - raise AssertionError("urlopen (transition or comment) must never be called in --dry-run") - - monkeypatch.setattr(msd.urllib.request, "urlopen", _boom) - - input_path = tmp_path / "shipped-stories.json" - input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") - - monkeypatch.setattr( - "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), "--dry-run", - "--workflow-id", str(WORKFLOW_ID), - "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], - ) - msd.main() - - out = json.loads(capsys.readouterr().out) - assert out["counts"]["comment_posted"] == 0 - assert out["counts"]["comment_failed"] == 0 - - -def test_dry_run_previews_the_comment_it_would_post(monkeypatch, tmp_path, capsys): - monkeypatch.delenv("SHORTCUT_API_TOKEN", raising=False) - monkeypatch.setattr( - msd.urllib.request, "urlopen", - lambda *a, **k: (_ for _ in ()).throw(AssertionError("urlopen must never be called in --dry-run")), - ) - - input_path = tmp_path / "shipped-stories.json" - input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") - - monkeypatch.setattr( - "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), "--dry-run", - "--workflow-id", str(WORKFLOW_ID), - "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], - ) - msd.main() - - err = capsys.readouterr().err - assert "would post comment on story 11111" in err - assert "6.111.0-prod.2" in err - assert "https://github.com/Sefaria/Sefaria-Project/pull/3644" in err - - -def test_comment_not_posted_with_no_comment_flag(monkeypatch, tmp_path): - monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") - - def _fake_urlopen(req, timeout=None): - if req.get_method() == "POST": - raise AssertionError("--no-comment must suppress the write-back comment entirely") - return _OKResponse() - - monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) - - input_path = tmp_path / "shipped-stories.json" - input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") - - monkeypatch.setattr( - "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), "--no-comment", - "--workflow-id", str(WORKFLOW_ID), - "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], - ) - msd.main() # must not raise -- proves no POST was attempted - - -def test_comment_api_failure_does_not_fail_the_run_and_is_reported(monkeypatch, tmp_path, capsys): - """A failed comment POST must never fail the run or roll back the - transition -- it's reported (warned + counted) and nothing else.""" - monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") - - import urllib.error - - def _fake_urlopen(req, timeout=None): - if req.get_method() == "PUT": - return _OKResponse() - raise urllib.error.HTTPError(req.full_url, 503, "Service Unavailable", None, None) - - monkeypatch.setattr(msd.urllib.request, "urlopen", _fake_urlopen) - - input_path = tmp_path / "shipped-stories.json" - input_path.write_text(json.dumps(RELEASE_INPUT_WITH_COMMITS), encoding="utf-8") - - monkeypatch.setattr( - "sys.argv", - ["mark_stories_deployed.py", "--input", str(input_path), - "--workflow-id", str(WORKFLOW_ID), - "--from-state-id", str(FROM_STATE), "--done-state-id", str(DONE_STATE)], - ) - # Must return normally -- a comment failure is never fatal. - msd.main() - - captured = capsys.readouterr() - out = json.loads(captured.out) - assert out["counts"]["transitioned"] == 1 - assert out["counts"]["comment_failed"] == 1 - assert out["comment_failed"] == [{"id": 11111, "error": "HTTP 503 Service Unavailable"}] - assert "Failed to post release comment on story 11111" in captured.err - - -def test_release_comment_text_falls_back_when_no_pr_reference_available(): - """A story whose carrying commit has no PR number (or whose release's - commits list doesn't mention it at all) must still get a comment -- - just without a PR link, not a crash.""" - data = {"version": "6.111.0-prod.2", "chart_version": "0.87.5-prod.1", - "release_date": "2026-08-31T07:17:36Z", "commits": []} - text = msd._release_comment_text({"id": 11111}, data, "Sefaria/Sefaria-Project") - assert "not available" in text - assert "6.111.0-prod.2" in text diff --git a/build/ci/tests/test_merge_release_backfill.py b/build/ci/tests/test_merge_release_backfill.py new file mode 100644 index 0000000000..c986520c65 --- /dev/null +++ b/build/ci/tests/test_merge_release_backfill.py @@ -0,0 +1,261 @@ +"""Tests for build/ci/merge_release_backfill.py. + +No network calls, no `git`/`gh` calls: this script is a plain, deterministic +JSON merge with no I/O beyond reading --shipped-stories-out / +--reconcile-report and writing --out. + +All story ids used below (11111, 22222, ...) are placeholders, not real +Shortcut story ids. +""" + +import json + +import pytest + +import merge_release_backfill as mrb + +SHIPPED_STORIES_BASE = { + "version": "6.111.0-prod.2", + "chart_version": "0.87.5-prod.1", + "release_date": "2026-08-31T07:17:36Z", + "story_ids": ["11111"], + "stories": [{"id": 11111, "name": "Found by shipped_stories.py's own scan"}], +} + + +def _hydrated_story(story_id, name="Backfilled Story"): + return { + "id": story_id, + "name": name, + "description": "A description", + "url": f"https://app.shortcut.com/org/story/{story_id}", + "workflow_id": 500000005, + "workflow_state_id": 500000045, + "story_type": "feature", + } + + +def _reconcile_report(shipped): + return {"prod_tag": "prod/6.111.0-prod.2+chart.0.87.5-prod.1", "applied": True, + "counts": {"total": len(shipped), "shipped": len(shipped), "pending": 0, "triage": 0}, + "shipped": shipped, "pending": [], "triage": []} + + +# --- backfill_stories_from_report: trusts hydrated_story's presence only - + +def test_backfill_stories_from_report_returns_only_entries_with_hydrated_story(): + report = _reconcile_report([ + {"id": 22222, "hydrated_story": _hydrated_story(22222)}, + {"id": 33333, "shipping_release_tag": "prod/earlier"}, # no hydrated_story -- earlier release + {"id": 44444}, # unresolvable release -- also no hydrated_story + ]) + backfill = mrb.backfill_stories_from_report(report) + assert [s["id"] for s in backfill] == [22222] + + +def test_backfill_stories_from_report_empty_shipped_bucket(): + report = _reconcile_report([]) + assert mrb.backfill_stories_from_report(report) == [] + + +def test_backfill_stories_from_report_missing_shipped_key_degrades_to_empty(): + assert mrb.backfill_stories_from_report({"prod_tag": "prod/1.0"}) == [] + + +# --- merge: dedup, additive, provenance ----------------------------------- + +def test_merge_adds_a_story_whose_derived_release_is_the_current_tag(): + data = dict(SHIPPED_STORIES_BASE, story_ids=list(SHIPPED_STORIES_BASE["story_ids"]), + stories=list(SHIPPED_STORIES_BASE["stories"])) + backfill = [_hydrated_story(22222)] + merged, added_ids = mrb.merge(data, backfill) + assert added_ids == ["22222"] + assert merged["story_ids"] == ["11111", "22222"] + assert [s["id"] for s in merged["stories"]] == [11111, 22222] + assert merged["stories_from_reconciliation_backfill"] == ["22222"] + + +def test_merge_does_not_add_a_story_from_an_earlier_release(): + """The merge function itself only ever sees what backfill_stories_from_ + report already filtered down to -- this test drives that filtering + step too, confirming an earlier-release entry never reaches merge().""" + data = dict(SHIPPED_STORIES_BASE, story_ids=list(SHIPPED_STORIES_BASE["story_ids"]), + stories=list(SHIPPED_STORIES_BASE["stories"])) + report = _reconcile_report([{"id": 22222, "shipping_release_tag": "prod/6.100.0-prod.1+chart.0.85.8-prod.1"}]) + backfill = mrb.backfill_stories_from_report(report) + merged, added_ids = mrb.merge(data, backfill) + assert added_ids == [] + assert merged["story_ids"] == ["11111"] + + +def test_merge_does_not_add_a_story_with_unresolvable_release(): + data = dict(SHIPPED_STORIES_BASE, story_ids=list(SHIPPED_STORIES_BASE["story_ids"]), + stories=list(SHIPPED_STORIES_BASE["stories"])) + report = _reconcile_report([{"id": 22222, "shipping_release_tag": None}]) + backfill = mrb.backfill_stories_from_report(report) + merged, added_ids = mrb.merge(data, backfill) + assert added_ids == [] + assert merged["story_ids"] == ["11111"] + + +def test_merge_no_duplicates_when_both_discovery_paths_find_the_same_story(): + """A story shipped_stories.py's own git-range/RC1 discovery ALSO + found (already present in story_ids/stories) must not be duplicated + when the reconciliation sweep independently finds it too.""" + data = dict(SHIPPED_STORIES_BASE, story_ids=list(SHIPPED_STORIES_BASE["story_ids"]), + stories=list(SHIPPED_STORIES_BASE["stories"])) + backfill = [_hydrated_story(11111, name="Same story, found twice")] + merged, added_ids = mrb.merge(data, backfill) + assert added_ids == [] + assert merged["story_ids"] == ["11111"] + assert len(merged["stories"]) == 1 + # The pre-existing entry (shipped_stories.py's own hydration) wins -- + # never overwritten by the backfill's version of the same story. + assert merged["stories"][0]["name"] == "Found by shipped_stories.py's own scan" + + +def test_merge_multiple_backfill_stories_all_added(): + data = dict(SHIPPED_STORIES_BASE, story_ids=list(SHIPPED_STORIES_BASE["story_ids"]), + stories=list(SHIPPED_STORIES_BASE["stories"])) + backfill = [_hydrated_story(22222), _hydrated_story(33333)] + merged, added_ids = mrb.merge(data, backfill) + assert added_ids == ["22222", "33333"] + assert merged["story_ids"] == ["11111", "22222", "33333"] + + +def test_merge_empty_backfill_is_a_no_op(): + data = dict(SHIPPED_STORIES_BASE, story_ids=list(SHIPPED_STORIES_BASE["story_ids"]), + stories=list(SHIPPED_STORIES_BASE["stories"])) + merged, added_ids = mrb.merge(data, []) + assert added_ids == [] + assert merged["story_ids"] == ["11111"] + assert merged["stories_from_reconciliation_backfill"] == [] + + +def test_merge_preserves_other_shipped_stories_json_fields(): + """version/chart_version/release_date and any other existing field + must survive the merge untouched -- this script only ever touches + story_ids/stories/stories_from_reconciliation_backfill.""" + data = dict(SHIPPED_STORIES_BASE, story_ids=list(SHIPPED_STORIES_BASE["story_ids"]), + stories=list(SHIPPED_STORIES_BASE["stories"])) + merged, _ = mrb.merge(data, [_hydrated_story(22222)]) + assert merged["version"] == "6.111.0-prod.2" + assert merged["chart_version"] == "0.87.5-prod.1" + assert merged["release_date"] == "2026-08-31T07:17:36Z" + + +# --- main(): end to end, reads two files, writes merged result ----------- + +def test_main_merges_current_release_backfill_end_to_end(tmp_path): + shipped_path = tmp_path / "shipped-stories.json" + shipped_path.write_text(json.dumps(SHIPPED_STORIES_BASE), encoding="utf-8") + + report_path = tmp_path / "reconcile-report.json" + report_path.write_text(json.dumps(_reconcile_report([ + {"id": 22222, "hydrated_story": _hydrated_story(22222)}, + {"id": 33333, "shipping_release_tag": "prod/earlier"}, + ])), encoding="utf-8") + + out_path = tmp_path / "shipped-stories.json" # default: overwrite in place + + import sys + old_argv = sys.argv + try: + sys.argv = [ + "merge_release_backfill.py", + "--shipped-stories-out", str(shipped_path), + "--reconcile-report", str(report_path), + ] + mrb.main() + finally: + sys.argv = old_argv + + merged = json.loads(out_path.read_text(encoding="utf-8")) + assert merged["story_ids"] == ["11111", "22222"] + assert 33333 not in [s.get("id") for s in merged["stories"]] + + +def test_main_writes_to_explicit_out_path_when_given(tmp_path): + shipped_path = tmp_path / "shipped-stories.json" + shipped_path.write_text(json.dumps(SHIPPED_STORIES_BASE), encoding="utf-8") + report_path = tmp_path / "reconcile-report.json" + report_path.write_text(json.dumps(_reconcile_report([])), encoding="utf-8") + explicit_out = tmp_path / "merged.json" + + import sys + old_argv = sys.argv + try: + sys.argv = [ + "merge_release_backfill.py", + "--shipped-stories-out", str(shipped_path), + "--reconcile-report", str(report_path), + "--out", str(explicit_out), + ] + mrb.main() + finally: + sys.argv = old_argv + + assert explicit_out.exists() + # The original --shipped-stories-out is untouched when --out is given. + original = json.loads(shipped_path.read_text(encoding="utf-8")) + assert original == SHIPPED_STORIES_BASE + + +def test_main_missing_shipped_stories_file_dies_cleanly(tmp_path): + report_path = tmp_path / "reconcile-report.json" + report_path.write_text(json.dumps(_reconcile_report([])), encoding="utf-8") + + import sys + old_argv = sys.argv + try: + sys.argv = [ + "merge_release_backfill.py", + "--shipped-stories-out", str(tmp_path / "does-not-exist.json"), + "--reconcile-report", str(report_path), + ] + with pytest.raises(SystemExit): + mrb.main() + finally: + sys.argv = old_argv + + +def test_main_missing_reconcile_report_dies_cleanly(tmp_path): + shipped_path = tmp_path / "shipped-stories.json" + shipped_path.write_text(json.dumps(SHIPPED_STORIES_BASE), encoding="utf-8") + + import sys + old_argv = sys.argv + try: + sys.argv = [ + "merge_release_backfill.py", + "--shipped-stories-out", str(shipped_path), + "--reconcile-report", str(tmp_path / "does-not-exist.json"), + ] + with pytest.raises(SystemExit): + mrb.main() + finally: + sys.argv = old_argv + + +# --- isolation guarantee: this module never touches the reconcile --------- +# --- report's shipped/pending/triage wholesale, and never imports -------- +# --- anything that could reach the network or Shortcut ------------------- + +def test_merge_release_backfill_module_never_imports_network_or_subprocess(): + """This script only ever reads two JSON files and writes one -- it + must never import urllib, requests, or subprocess. It has no business + calling out to anything; if it ever needed to, that would be a sign + the isolation this script exists to preserve had already broken.""" + import ast + import inspect + + tree = ast.parse(inspect.getsource(mrb)) + imported_names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported_names.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported_names.add(node.module.split(".")[0]) + + forbidden = {"urllib", "requests", "subprocess", "socket", "http"} + assert not (imported_names & forbidden), imported_names & forbidden diff --git a/build/ci/tests/test_reconcile_deploy_ready.py b/build/ci/tests/test_reconcile_deploy_ready.py index 901db441c2..0529efc02e 100644 --- a/build/ci/tests/test_reconcile_deploy_ready.py +++ b/build/ci/tests/test_reconcile_deploy_ready.py @@ -30,17 +30,19 @@ OTHER_WORKFLOW_STATE_ID = 500000900 -def _pr(number, merged=True, repository_id=SEFARIA_REPO_ID, target_branch_name="master"): +def _pr(number, merged=True, repository_id=SEFARIA_REPO_ID, target_branch_name="master", + branch_name="feature/some-branch"): return { "number": number, "merged": merged, "repository_id": repository_id, "target_branch_name": target_branch_name, + "branch_name": branch_name, } def _story(story_id, name="Story", workflow_id=STANDARD_WORKFLOW_ID, workflow_state_id=DEPLOY_READY_STATE_ID, - pull_requests=None, branches=None, description=None, comments=None): + pull_requests=None, branches=None, description=None, comments=None, story_type="feature"): return { "id": story_id, "name": name, @@ -51,6 +53,7 @@ def _story(story_id, name="Story", workflow_id=STANDARD_WORKFLOW_ID, workflow_st "branches": branches or [], "description": description, "comments": comments or [], + "story_type": story_type, } @@ -82,10 +85,10 @@ def test_gather_linked_prs_ignores_prs_with_no_number(): assert rdr.gather_linked_prs(story) == [] -# --- qualifying_prs: the three PR-level guards --------------------------- +# --- qualifying_prs: the four PR-level guards (shortcut_pr_guards.py) ---- def test_qualifying_prs_wrong_repo_guard(): - """Guard #1: a PR linked from a DIFFERENT repo must never count as + """Guard: a PR linked from a DIFFERENT repo must never count as evidence a Sefaria-Project story shipped, even if it's merged and targets 'master' -- resolving it against Sefaria-Project would find an unrelated PR that happens to share the number.""" @@ -93,8 +96,8 @@ def test_qualifying_prs_wrong_repo_guard(): assert rdr.qualifying_prs(prs) == [] -def test_qualifying_prs_promotion_pr_guard(): - """Guard #2: a PR targeting anything other than 'master' (e.g. a +def test_qualifying_prs_promotion_pr_target_branch_guard(): + """Guard: a PR targeting anything other than 'master' (e.g. a preprod->prod or master->preprod promotion PR some stories link instead of the real feature PR) must not qualify.""" prs = [_pr(3551, target_branch_name="prod")] @@ -103,14 +106,32 @@ def test_qualifying_prs_promotion_pr_guard(): assert rdr.qualifying_prs(prs2) == [] +def test_qualifying_prs_promotion_pr_head_branch_guard_preprod_to_master(): + """Guard (regression): a promotion PR merging preprod INTO master + passes the repo/merged/target-branch guards cleanly -- its target + really is 'master'. Only the HEAD branch (branch_name) reveals it's a + promotion merge, not a feature PR. This is the exact live false + positive that was found and fixed: a story was classified `shipped` on + the strength of a PR shaped exactly like this.""" + prs = [_pr(3677, target_branch_name="master", branch_name="preprod")] + assert rdr.qualifying_prs(prs) == [] + + +def test_qualifying_prs_promotion_pr_head_branch_guard_master_to_preprod(): + """The other promotion direction is already caught by the target-branch + guard, but the head-branch guard must reject it too, independently.""" + prs = [_pr(3698, target_branch_name="preprod", branch_name="master")] + assert rdr.qualifying_prs(prs) == [] + + def test_qualifying_prs_unmerged_pr_guard(): - """Guard #3: an open or closed-without-merging PR proves nothing.""" + """Guard: an open or closed-without-merging PR proves nothing.""" prs = [_pr(3397, merged=False)] assert rdr.qualifying_prs(prs) == [] -def test_qualifying_prs_accepts_a_pr_passing_all_three_guards(): - prs = [_pr(3606)] +def test_qualifying_prs_accepts_a_normal_feature_pr_passing_all_four_guards(): + prs = [_pr(3606, branch_name="feature/some-fix")] assert rdr.qualifying_prs(prs) == prs @@ -119,14 +140,15 @@ def test_qualifying_prs_filters_mixed_list_keeping_only_the_valid_one(): _pr(224, repository_id=OTHER_REPO_ID), _pr(3551, target_branch_name="prod"), _pr(3397, merged=False), - _pr(3606), + _pr(3677, target_branch_name="master", branch_name="preprod"), + _pr(3606, branch_name="feature/some-fix"), ] assert [p["number"] for p in rdr.qualifying_prs(prs)] == [3606] def test_qualifying_prs_custom_repo_and_target_branch_args(): """--repo/--target-branch overrides are threaded through, not hardcoded.""" - prs = [_pr(1, repository_id=999, target_branch_name="main")] + prs = [_pr(1, repository_id=999, target_branch_name="main", branch_name="feature/some-fix")] assert rdr.qualifying_prs(prs, repo_id=999, target_branch="main") == prs assert rdr.qualifying_prs(prs, repo_id=SEFARIA_REPO_ID, target_branch="master") == [] @@ -189,6 +211,15 @@ def test_diagnose_linked_pr_reports_only_the_specific_guard_that_failed(): assert failed == ["wrong target branch ('preprod', expected 'master')"] +def test_diagnose_linked_pr_reports_promotion_head_branch_guard(): + """A promotion PR (preprod -> master) passes merged/repo/target-branch + but must be flagged for its head branch -- the diagnostic must surface + the SAME reason qualifying_prs silently rejects it for.""" + pr = _pr(3677, target_branch_name="master", branch_name="preprod") + failed = rdr._diagnose_linked_pr(pr, SEFARIA_REPO_ID, "master") + assert failed == ["promotion PR (head branch 'preprod' is a long-lived environment branch)"] + + def test_triage_context_extracts_description_and_comment_text(): story = _story( 11111, description="Some story description", @@ -274,6 +305,7 @@ def test_classify_stories_qualifying_story_becomes_a_candidate(): def test_classify_candidates_ancestor_true_is_shipped(monkeypatch): monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: True) + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: None) story = _story(11111, pull_requests=[_pr(3606)]) candidates = [(story, [_pr(3606)])] shipped, pending = rdr.classify_candidates(candidates, {3606: "abc123"}, "prod/1.0") @@ -325,6 +357,7 @@ def test_classify_candidates_any_qualifying_pr_in_prod_is_enough(monkeypatch): """A story with two qualifying PRs, only one of which is in prod, still ships -- 'at least one' per the spec.""" monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: oid == "in-prod-oid") + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: None) story = _story(11111, pull_requests=[_pr(1), _pr(2)]) candidates = [(story, [_pr(1), _pr(2)])] @@ -335,6 +368,85 @@ def test_classify_candidates_any_qualifying_pr_in_prod_is_enough(monkeypatch): assert shipped[0]["qualifying_prs"] == [1, 2] +# --- classify_candidates: shipping_release_tag / hydrated_story ---------- +# --- (the "backfilled story that shipped in THIS release" fix) ----------- + +def test_classify_candidates_shipping_release_tag_equal_to_current_attaches_hydrated_story(monkeypatch): + """When the derived TRUE shipping release equals the CURRENT prod tag, + the entry gets a hydrated_story sub-object -- the ONLY signal + merge_release_backfill.py uses to fold a backfilled story into + shipped-stories.json.""" + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: True) + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: "prod/1.0") + story = _story(11111, name="Some Story", pull_requests=[_pr(3606)], description="A description") + candidates = [(story, [_pr(3606)])] + shipped, pending = rdr.classify_candidates(candidates, {3606: "abc"}, "prod/1.0") + assert shipped[0]["shipping_release_tag"] == "prod/1.0" + hydrated = shipped[0]["hydrated_story"] + assert hydrated == { + "id": 11111, + "name": "Some Story", + "description": "A description", + "url": f"https://app.shortcut.com/org/story/11111", + "workflow_id": STANDARD_WORKFLOW_ID, + "workflow_state_id": DEPLOY_READY_STATE_ID, + "story_type": "feature", + } + + +def test_classify_candidates_shipping_release_tag_earlier_than_current_excludes_hydrated_story(monkeypatch): + """A story that shipped in an EARLIER release must never carry + hydrated_story -- that's exactly the 'old features shipped today' + leak this whole design rejects.""" + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: True) + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: "prod/0.9-earlier") + story = _story(11111, pull_requests=[_pr(3606)]) + candidates = [(story, [_pr(3606)])] + shipped, pending = rdr.classify_candidates(candidates, {3606: "abc"}, "prod/1.0") + assert shipped[0]["shipping_release_tag"] == "prod/0.9-earlier" + assert "hydrated_story" not in shipped[0] + + +def test_classify_candidates_unresolvable_shipping_release_excludes_hydrated_story(monkeypatch): + """resolve_shipping_release_tag returning None (unresolvable) must + also never attach hydrated_story -- fail closed, never guess a story + into the announcement.""" + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: True) + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: None) + story = _story(11111, pull_requests=[_pr(3606)]) + candidates = [(story, [_pr(3606)])] + shipped, pending = rdr.classify_candidates(candidates, {3606: "abc"}, "prod/1.0") + assert shipped[0]["shipping_release_tag"] is None + assert "hydrated_story" not in shipped[0] + + +def test_classify_candidates_pending_entries_never_carry_shipping_release_tag(monkeypatch): + """shipping_release_tag/hydrated_story are shipped-only concepts -- a + pending story (nothing in prod yet) has no shipping release to derive + at all.""" + monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: False) + + def _boom(oid): + raise AssertionError("resolve_shipping_release_tag must not be called for a pending story") + + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", _boom) + story = _story(11111, pull_requests=[_pr(3670)]) + candidates = [(story, [_pr(3670)])] + shipped, pending = rdr.classify_candidates(candidates, {3670: "def"}, "prod/1.0") + assert shipped == [] + assert "shipping_release_tag" not in pending[0] + assert "hydrated_story" not in pending[0] + + +def test_story_for_release_notes_matches_shipped_stories_hydrated_shape(): + story = _story(11111, name="A Story", description="Desc", story_type="bug") + hydrated = rdr._story_for_release_notes(story) + assert set(hydrated.keys()) == { + "id", "name", "description", "url", "workflow_id", "workflow_state_id", "story_type", + } + assert hydrated["url"] == story["app_url"] + + # --- is_ancestor_of_prod: exit-code interpretation ----------------------- def test_is_ancestor_of_prod_true_on_exit_code_0(monkeypatch): @@ -445,17 +557,16 @@ class _Proc: # --- End-to-end main(): dry-run is the default and mutates nothing ------- def _make_main_env(monkeypatch, tmp_path, stories, prod_tag="prod/1.0", oid_by_pr=None, - ancestor_result=True, argv_extra=None, shipping_tag=None, - mock_post_comment=True): + ancestor_result=True, argv_extra=None, shipping_tag=None): """Wire main() end-to-end with every I/O boundary mocked: Shortcut - search, gh merge-commit lookup, git ancestry, the git-based - shipping-release lookup, and (by default) the comment POST itself -- - mirroring _run_main_with_commits in test_shipped_stories.py. - resolve_shipping_release_tag defaults to a plain lambda returning - `shipping_tag` (None unless overridden) rather than shelling out to - real git, so tests that don't care about that specific behavior stay - hermetic; the dedicated tests for it below monkeypatch it themselves - when they need to exercise its own git-shelling logic.""" + search, gh merge-commit lookup, git ancestry, and the git-based + shipping-release lookup -- mirroring _run_main_with_commits in + test_shipped_stories.py. resolve_shipping_release_tag defaults to a + plain lambda returning `shipping_tag` (None unless overridden) rather + than shelling out to real git, so tests that don't care about that + specific behavior stay hermetic; the dedicated tests for it below + monkeypatch it themselves when they need to exercise its own + git-shelling logic.""" monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") monkeypatch.setattr(rdr, "search_deploy_ready_stories", lambda token: stories) monkeypatch.setattr(rdr, "resolve_default_prod_tag", lambda: prod_tag) @@ -466,11 +577,6 @@ def _make_main_env(monkeypatch, tmp_path, stories, prod_tag="prod/1.0", oid_by_p ) monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: ancestor_result) monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: shipping_tag) - if mock_post_comment: - monkeypatch.setattr( - rdr.shortcut_comment, "post_story_comment", - lambda story_id, text, token: (story_id, True, None), - ) argv = ["reconcile_deploy_ready.py"] + (argv_extra or []) monkeypatch.setattr("sys.argv", argv) @@ -492,13 +598,10 @@ def _boom(*args, **kwargs): def test_main_apply_transitions_shipped_stories(monkeypatch, tmp_path, capsys): - """--no-comment here to keep this test scoped to transition mechanics - only -- the write-back comment behavior has its own dedicated tests - below.""" story = _story(11111, pull_requests=[_pr(3606)]) _make_main_env( monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - argv_extra=["--apply", "--no-comment"], + argv_extra=["--apply"], ) calls = [] @@ -542,10 +645,7 @@ def test_main_writes_out_json_report(monkeypatch, tmp_path): rdr.main() report = json.loads(out_path.read_text(encoding="utf-8")) - assert report["counts"] == { - "total": 2, "shipped": 1, "pending": 0, "triage": 1, - "comment_posted": 0, "comment_failed": 0, - } + assert report["counts"] == {"total": 2, "shipped": 1, "pending": 0, "triage": 1} assert report["applied"] is False assert [s["id"] for s in report["shipped"]] == [11111] assert [s["id"] for s in report["triage"]] == [22222] @@ -560,10 +660,7 @@ def test_main_pending_bucket_when_qualifying_pr_not_yet_in_prod(monkeypatch, tmp ) rdr.main() report = json.loads(out_path.read_text(encoding="utf-8")) - assert report["counts"] == { - "total": 1, "shipped": 0, "pending": 1, "triage": 0, - "comment_posted": 0, "comment_failed": 0, - } + assert report["counts"] == {"total": 1, "shipped": 0, "pending": 1, "triage": 0} assert report["pending"][0]["id"] == 11111 @@ -609,6 +706,7 @@ def _boom_default_tag(): monkeypatch.setattr(rdr, "resolve_default_prod_tag", _boom_default_tag) monkeypatch.setattr(rdr, "fetch_merge_oids", lambda pr_numbers, repo, max_workers=8: {3606: "abc"}) monkeypatch.setattr(rdr, "is_ancestor_of_prod", lambda oid, tag: tag == "prod/explicit-tag") + monkeypatch.setattr(rdr, "resolve_shipping_release_tag", lambda oid: None) monkeypatch.setattr( "sys.argv", ["reconcile_deploy_ready.py", "--prod-tag", "prod/explicit-tag", "--out", str(out_path)], @@ -682,176 +780,74 @@ class _Proc: assert "WARNING" in capsys.readouterr().err -# --- write-back release comment: posted / not posted / dry-run preview --- -# --- / --no-comment / API failure / honest degrade (regression coverage -- -# --- for the promotion-PR-style "old features shipped today" mistake, -- -# --- just in a Shortcut comment instead of a Slack post) ----------------- - -def test_main_posts_comment_after_a_real_transition(monkeypatch, tmp_path): - story = _story(11111, pull_requests=[_pr(3606)]) - calls = [] - - _make_main_env( - monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - argv_extra=["--apply"], shipping_tag="prod/6.100.0-prod.1+chart.0.85.8-prod.1", - mock_post_comment=False, - ) - monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, True, None)) - monkeypatch.setattr( - rdr.shortcut_comment, "post_story_comment", - lambda story_id, text, token: (calls.append((story_id, text)), story_id, True, None)[1:], - ) - - rdr.main() - - assert len(calls) == 1 - posted_id, text = calls[0] - assert posted_id == 11111 - assert "prod/6.100.0-prod.1+chart.0.85.8-prod.1" in text - assert "not part of today's release" in text - assert "https://github.com/Sefaria/Sefaria-Project/pull/3606" in text - # Must NOT name the CURRENT prod tag as if the story shipped in it. - assert "prod/1.0" not in text - - -def test_main_does_not_post_comment_when_transition_fails(monkeypatch, tmp_path): - story = _story(11111, pull_requests=[_pr(3606)]) - - def _boom_post(*args, **kwargs): - raise AssertionError("no comment must be posted for a story whose transition failed") - - _make_main_env( - monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - argv_extra=["--apply"], mock_post_comment=False, - ) - monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, False, "HTTP 500 Internal Server Error")) - monkeypatch.setattr(rdr.shortcut_comment, "post_story_comment", _boom_post) - - with pytest.raises(SystemExit): - rdr.main() # transition failure still exits non-zero - -def test_main_does_not_post_comment_in_dry_run(monkeypatch, tmp_path): - story = _story(11111, pull_requests=[_pr(3606)]) +# --- main(): shipping_release_tag / hydrated_story end-to-end ------------ - def _boom_post(*args, **kwargs): - raise AssertionError("no comment must be posted in dry-run") - - _make_main_env( - monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - mock_post_comment=False, # dry-run (no --apply): must never even try to POST - ) - monkeypatch.setattr(rdr.shortcut_comment, "post_story_comment", _boom_post) - rdr.main() # must not raise -- proves no POST was attempted - - -def test_main_dry_run_report_previews_the_comment_it_would_post(monkeypatch, tmp_path): - story = _story(11111, pull_requests=[_pr(3606)]) +def test_main_report_attaches_hydrated_story_when_shipping_release_matches_current(monkeypatch, tmp_path): + """End-to-end: a shipped story whose derived release equals the + CURRENT prod tag gets a hydrated_story entry in the report -- the + signal a separate merge step uses to fold it into today's release + notes.""" + story = _story(11111, name="Some Story", pull_requests=[_pr(3606)], description="A description") out_path = tmp_path / "report.json" _make_main_env( monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - shipping_tag="prod/6.100.0-prod.1+chart.0.85.8-prod.1", + prod_tag="prod/1.0", shipping_tag="prod/1.0", argv_extra=["--out", str(out_path)], ) rdr.main() - report = json.loads(out_path.read_text(encoding="utf-8")) - assert report["counts"]["comment_posted"] == 0 - would_comment = report["shipped"][0]["would_comment"] - assert "prod/6.100.0-prod.1+chart.0.85.8-prod.1" in would_comment + entry = report["shipped"][0] + assert entry["shipping_release_tag"] == "prod/1.0" + assert entry["hydrated_story"]["id"] == 11111 + assert entry["hydrated_story"]["name"] == "Some Story" -def test_main_no_comment_flag_suppresses_posting(monkeypatch, tmp_path): - story = _story(11111, pull_requests=[_pr(3606)]) - - def _boom_post(*args, **kwargs): - raise AssertionError("--no-comment must suppress the write-back comment entirely") - - _make_main_env( - monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - argv_extra=["--apply", "--no-comment"], mock_post_comment=False, - ) - monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, True, None)) - monkeypatch.setattr(rdr.shortcut_comment, "post_story_comment", _boom_post) - - rdr.main() # must not raise -- proves no POST was attempted - - -def test_main_no_comment_flag_suppresses_dry_run_preview_too(monkeypatch, tmp_path): +def test_main_report_excludes_hydrated_story_for_an_earlier_release(monkeypatch, tmp_path): story = _story(11111, pull_requests=[_pr(3606)]) out_path = tmp_path / "report.json" _make_main_env( monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - argv_extra=["--no-comment", "--out", str(out_path)], + prod_tag="prod/9.9.9-prod.1+chart.9.9.9-prod.1", shipping_tag="prod/6.100.0-prod.1+chart.0.85.8-prod.1", + argv_extra=["--out", str(out_path)], ) rdr.main() report = json.loads(out_path.read_text(encoding="utf-8")) - assert "would_comment" not in report["shipped"][0] - - -def test_main_comment_api_failure_does_not_fail_the_run_and_is_reported(monkeypatch, tmp_path, capsys): - """A failed comment POST must never fail the run or roll back the - transition -- it's reported (warned + counted) and nothing else.""" - story = _story(11111, pull_requests=[_pr(3606)]) - out_path = tmp_path / "report.json" - - _make_main_env( - monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - argv_extra=["--apply", "--out", str(out_path)], mock_post_comment=False, - ) - monkeypatch.setattr(rdr, "transition_story", lambda sid, done, token: (sid, True, None)) - monkeypatch.setattr( - rdr.shortcut_comment, "post_story_comment", - lambda story_id, text, token: (story_id, False, "HTTP 503 Service Unavailable"), - ) - - rdr.main() # must return normally -- a comment failure is never fatal - - report = json.loads(out_path.read_text(encoding="utf-8")) - assert report["counts"]["shipped"] == 1 - assert report["counts"]["comment_failed"] == 1 - assert report["comment_failed"] == [{"id": 11111, "error": "HTTP 503 Service Unavailable"}] - assert "Failed to post release comment on story 11111" in capsys.readouterr().err + entry = report["shipped"][0] + assert entry["shipping_release_tag"] == "prod/6.100.0-prod.1+chart.0.85.8-prod.1" + assert "hydrated_story" not in entry -def test_main_comment_names_the_contains_derived_release_not_the_current_one(monkeypatch, tmp_path): - """Regression coverage for the exact mistake this feature exists to - avoid: a story backfilled by this sweep shipped in an EARLIER release, - and the comment must name THAT release (resolved via - resolve_shipping_release_tag's `--contains` lookup), never the current - --prod-tag / today's release.""" +def test_main_report_excludes_hydrated_story_when_release_unresolvable(monkeypatch, tmp_path): story = _story(11111, pull_requests=[_pr(3606)]) out_path = tmp_path / "report.json" _make_main_env( monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - prod_tag="prod/9.9.9-prod.1+chart.9.9.9-prod.1", # "today's" release - shipping_tag="prod/6.100.0-prod.1+chart.0.85.8-prod.1", # the TRUE, earlier release + prod_tag="prod/9.9.9-prod.1+chart.9.9.9-prod.1", shipping_tag=None, argv_extra=["--out", str(out_path)], ) rdr.main() report = json.loads(out_path.read_text(encoding="utf-8")) - would_comment = report["shipped"][0]["would_comment"] - assert "prod/6.100.0-prod.1+chart.0.85.8-prod.1" in would_comment - assert "prod/9.9.9-prod.1+chart.9.9.9-prod.1" not in would_comment + entry = report["shipped"][0] + assert entry["shipping_release_tag"] is None + assert "hydrated_story" not in entry -def test_main_comment_degrades_honestly_when_release_tag_lookup_fails(monkeypatch, tmp_path): - """When resolve_shipping_release_tag can't determine the true release - (returns None -- e.g. a shallow checkout or a genuine history gap), the - comment must degrade to naming the current prod tag as "present as of" - language and the PR -- it must NEVER guess or imply a specific - release.""" +def test_main_never_posts_to_shortcut_beyond_the_state_transition(monkeypatch, tmp_path): + """Regression guard for the removed write-back comment feature: even + on a real --apply run, the only Shortcut mutation this script performs + is the workflow_state_id PUT (transition_story) -- nothing else calls + out to Shortcut.""" story = _story(11111, pull_requests=[_pr(3606)]) - out_path = tmp_path / "report.json" _make_main_env( monkeypatch, tmp_path, [story], oid_by_pr={3606: "abc"}, ancestor_result=True, - prod_tag="prod/9.9.9-prod.1+chart.9.9.9-prod.1", - shipping_tag=None, # lookup failed / inconclusive - argv_extra=["--out", str(out_path)], + argv_extra=["--apply"], + ) + calls = [] + monkeypatch.setattr( + rdr, "transition_story", + lambda sid, done, token: (calls.append(sid), sid, True, None)[1:], ) rdr.main() - report = json.loads(out_path.read_text(encoding="utf-8")) - would_comment = report["shipped"][0]["would_comment"] - assert "present in production as of prod/9.9.9-prod.1+chart.9.9.9-prod.1" in would_comment - assert "could not be determined" in would_comment - assert "https://github.com/Sefaria/Sefaria-Project/pull/3606" in would_comment + assert calls == [11111] + assert not hasattr(rdr, "shortcut_comment") diff --git a/build/ci/tests/test_shipped_stories.py b/build/ci/tests/test_shipped_stories.py index 8e0bfccc37..43144762b5 100644 --- a/build/ci/tests/test_shipped_stories.py +++ b/build/ci/tests/test_shipped_stories.py @@ -388,7 +388,8 @@ def test_fetch_story_workflow_id_defaults_to_none_when_absent(monkeypatch): # Shortcut story ids and are not covered by that convention.) -def _story_with_linked_pr(story_id, pr_number, merged=True, repository_id=500000103, target_branch_name="master"): +def _story_with_linked_pr(story_id, pr_number, merged=True, repository_id=500000103, target_branch_name="master", + branch_name="feature/some-branch"): """Minimal Shortcut search-result story payload carrying ONE linked PR -- enough for fetch_story_by_pr_link's guard re-check (shortcut_pr_guards.gather_linked_prs / passes_pr_guards) to find and @@ -402,6 +403,7 @@ def _story_with_linked_pr(story_id, pr_number, merged=True, repository_id=500000 "merged": merged, "repository_id": repository_id, "target_branch_name": target_branch_name, + "branch_name": branch_name, }], } @@ -464,6 +466,56 @@ def test_fetch_story_by_pr_link_unmerged_pr_guard_rejects(monkeypatch, capsys): assert "does not pass the shipping-evidence guards" in capsys.readouterr().err +def test_fetch_story_by_pr_link_promotion_head_branch_guard_rejects_preprod_to_master(monkeypatch, capsys): + """Guard #4 regression: a promotion PR merging preprod INTO master + passes guards 1-3 cleanly (merged, right repo, target IS master) -- + only the HEAD branch reveals it's a promotion merge, not a feature PR. + This is the exact live false positive that was found: a story was + classified shipped on the strength of a PR shaped exactly like this.""" + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse( + {"data": [_story_with_linked_pr(66666, "3677", target_branch_name="master", branch_name="preprod")], + "total": 1} + ), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("3677", "fake-token-for-tests") + assert story_id is None + assert "does not pass the shipping-evidence guards" in capsys.readouterr().err + + +def test_fetch_story_by_pr_link_promotion_head_branch_guard_rejects_master_to_preprod(monkeypatch, capsys): + """The other promotion direction (master -> preprod) is already caught + by guard #3 (target branch must be master) -- but guard #4 must reject + it too, independently, since the two checks are not redundant (a + linked PR could in principle target 'master' while also having a + long-lived head branch for some other reason).""" + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse( + {"data": [_story_with_linked_pr(66666, "3698", target_branch_name="master", branch_name="master")], + "total": 1} + ), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("3698", "fake-token-for-tests") + assert story_id is None + assert "does not pass the shipping-evidence guards" in capsys.readouterr().err + + +def test_fetch_story_by_pr_link_normal_feature_pr_still_passes_all_four_guards(monkeypatch): + """A real feature PR (merged, right repo, targets master, head branch + is an ordinary feature branch) must still be adopted -- guard #4 must + not be so broad it rejects legitimate PRs.""" + monkeypatch.setattr( + ss.urllib.request, "urlopen", + lambda req, timeout=None: _FakeShortcutResponse( + {"data": [_story_with_linked_pr(66666, "3606", branch_name="feature/some-fix")], "total": 1} + ), + ) + pr_number, story_id = ss.fetch_story_by_pr_link("3606", "fake-token-for-tests") + assert story_id == "66666" + + def test_fetch_story_by_pr_link_no_match_returns_none_quietly(monkeypatch, capsys): monkeypatch.setattr( ss.urllib.request, "urlopen", diff --git a/build/ci/tests/test_triage_explainer.py b/build/ci/tests/test_triage_explainer.py index b7984139b2..9c7bac69cc 100644 --- a/build/ci/tests/test_triage_explainer.py +++ b/build/ci/tests/test_triage_explainer.py @@ -16,14 +16,13 @@ FULL_REPORT = { "prod_tag": "prod/7.1.3-prod.1+chart.0.88.2-prod.1", "applied": False, - "counts": { - "total": 3, "shipped": 1, "pending": 1, "triage": 1, - "comment_posted": 0, "comment_failed": 0, - }, + "counts": {"total": 3, "shipped": 1, "pending": 1, "triage": 1}, "shipped": [ {"id": 11111, "name": "Shipped story", "url": "https://app.shortcut.com/org/story/11111", "shipped_via_prs": [3606], "qualifying_prs": [3606], "transitioned": True, - "would_comment": "should never leak into the explainer's input"}, + "shipping_release_tag": "prod/7.1.3-prod.1+chart.0.88.2-prod.1", + "hydrated_story": {"id": 11111, "name": "Shipped story", "description": "should never leak", + "url": "https://app.shortcut.com/org/story/11111"}}, ], "pending": [ {"id": 22222, "name": "Pending story", "url": "https://app.shortcut.com/org/story/22222", @@ -35,8 +34,6 @@ "linked_prs": [{"number": 3698, "failed_guards": ["wrong target branch ('preprod', expected 'master')"]}], "description": "A story description", "comments": ["why is this stuck?"]}, ], - "comment_posted": [], - "comment_failed": [], } @@ -47,7 +44,7 @@ def test_extract_triage_only_excludes_shipped_and_pending_entirely(): assert "shipped" not in result assert "pending" not in result # Not just absent as top-level keys -- the shipped/pending story data - # itself (ids, names, would_comment text) must not appear anywhere in + # itself (ids, names, hydrated_story text) must not appear anywhere in # the serialized output. serialized = json.dumps(result) assert "11111" not in serialized @@ -55,11 +52,9 @@ def test_extract_triage_only_excludes_shipped_and_pending_entirely(): assert "should never leak" not in serialized -def test_extract_triage_only_excludes_applied_and_comment_bookkeeping(): +def test_extract_triage_only_excludes_applied_and_counts(): result = te.extract_triage_only(FULL_REPORT) assert "applied" not in result - assert "comment_posted" not in result - assert "comment_failed" not in result assert "counts" not in result # counts.shipped/pending would otherwise leak bucket sizes diff --git a/build/ci/triage_explainer.py b/build/ci/triage_explainer.py index 538f5cf0cc..b17098f2bc 100644 --- a/build/ci/triage_explainer.py +++ b/build/ci/triage_explainer.py @@ -57,9 +57,7 @@ # top-level keys carelessly. Listed explicitly (rather than an # allow-only-"triage" approach implemented by construction below) as a # second, redundant line of defense -- see extract_triage_only. -EXCLUDED_REPORT_KEYS = frozenset({ - "shipped", "pending", "applied", "comment_posted", "comment_failed", -}) +EXCLUDED_REPORT_KEYS = frozenset({"shipped", "pending", "applied"}) def die(message: str) -> None: From 9a05efe82ccad48e9852b96d32065d4086b87364 Mon Sep 17 00:00:00 2001 From: Yotam Fromm Date: Mon, 7 Sep 2026 10:55:33 +0300 Subject: [PATCH 5/6] chore(ci): trim release-pipeline comments to essentials Trims module docstrings, function docstrings, and inline comments across the release-notes pipeline scripts, the workflow YAML, and the README down to what's needed operationally. No behavior change: only comments, docstrings, and markdown were touched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KJqBksQqzHYs3F54Y4tJRX --- .github/workflows/prod-release-notes.yaml | 196 +------- build/ci/README-prod-release-notes.md | 569 +++------------------- build/ci/mark_stories_deployed.py | 81 +-- build/ci/merge_release_backfill.py | 84 +--- build/ci/reconcile_deploy_ready.py | 318 ++---------- build/ci/shipped_stories.py | 287 ++--------- build/ci/shortcut_pr_guards.py | 79 +-- build/ci/triage_explainer.py | 86 +--- 8 files changed, 207 insertions(+), 1493 deletions(-) diff --git a/.github/workflows/prod-release-notes.yaml b/.github/workflows/prod-release-notes.yaml index def94a4fe3..ca94a99582 100644 --- a/.github/workflows/prod-release-notes.yaml +++ b/.github/workflows/prod-release-notes.yaml @@ -1,72 +1,16 @@ name: "Prod Release Notes" -# Fires when the prod rollout-complete AnalysisTemplate (see -# helm-chart/sefaria/templates/analysistemplate/rollout-complete.yaml) confirms -# Argo Rollouts' post-promotion analysis passed for prod — i.e. the new -# revision's pods are up and its health checks are green. That job relays a -# repository_dispatch here, carrying the deployed `version` and `chartVersion`. -# -# From there this workflow is a strict pipeline of deterministic scripts, with -# two steps handed to an LLM (release-notes prose, and an OPT-IN triage -# explanation -- see step 5 below): -# 1. build/ci/shipped_stories.py resolves the prod/* tag range for that -# version (chartVersion disambiguates a chart-only rollout, where several -# tags share the same app version), walks git log between the tags, and -# hydrates the Shortcut stories those commits reference. -# 2. build/ci/mark_stories_deployed.py moves each of those stories from -# Deploy Ready -> Done via the Shortcut API. It only ever writes that -# state -- no comment, no other annotation. A failure here (missing -# token, API error, nothing to move) is surfaced as a warning but never -# blocks steps 3-7 — see "Mark shipped stories as deployed" below. -# 3. build/ci/reconcile_deploy_ready.py sweeps EVERY non-archived Deploy -# Ready story org-wide (not just this release's commit range) and -# transitions any whose linked PR already reached prod, regardless of -# which release actually shipped it -- same "state only" posture as -# step 2. This backfills stories a prior release's git-range-scoped run -# never revisited (RC2) — see that script's own docstring for the four -# PR-level guards (+ one story-level workflow/state guard) it applies. -# Its report is written to $RUNNER_TEMP and NEVER read directly by -# steps 6-7: most of what it backfills shipped in EARLIER releases, and -# leaking that wholesale into today's announcement would have Slack -# claim old features shipped today. A failure here is warned/alerted -# the same way step 2's is, and never blocks steps 4-7. -# 4. build/ci/merge_release_backfill.py folds the ONE, pre-filtered -# exception into shipped-stories.json: a story step 3 backfilled whose -# TRUE shipping release (resolved there, from git) equals THIS run's -# own current release, not an earlier one -- that story really did -# just ship today and belongs in the announcement. This is the only -# channel through which anything from step 3's report ever reaches -# shipped-stories.json, and the decision was already made -# deterministically upstream; this step is a pure mechanical merge. -# 5. build/ci/triage_explainer.py + a headless `claude -p` run (OPT-IN, -# off by default -- see "Explain Deploy Ready triage backlog" below) -# proposes a short, labeled HYPOTHESIS for why each story in step 3's -# triage bucket is stuck, and a suggested next action for a human. It -# sees ONLY the triage bucket (never shipped/pending, structurally — -# see triage_explainer.py's own docstring), never mutates Shortcut, -# and its output never reaches steps 6-7 either. -# 6. The sefaria-release-notes skill (.claude/skills/sefaria-release-notes/) -# reads shipped-stories.json (now including step 4's merge, if any) and -# writes prose only — it does not talk to GitHub or Shortcut. -# 7. scripts/post_to_slack.py posts both generated files to Slack. -# -# Manual setup this depends on: see build/ci/README-prod-release-notes.md +# Fires when the prod rollout-complete AnalysisTemplate confirms the +# post-promotion analysis passed, via repository_dispatch carrying +# `version` and `chartVersion`. See build/ci/README-prod-release-notes.md. on: repository_dispatch: types: [prod-rollout-succeeded] - # repository_dispatch only ever runs the DEFAULT branch's copy of a - # workflow, so without workflow_dispatch there is no way to exercise this - # path at all before a change here merges, and no way to re-run it for a - # past release (e.g. after fixing a bug in one of the scripts below). + # workflow_dispatch lets this be re-run manually / tested off the default branch. workflow_dispatch: inputs: - # rollout-complete.yaml strips a leading "v" before building the - # repository_dispatch payload, so `inputs.version` is expected bare - # too (e.g. 6.111.0-prod.2, not v6.111.0-prod.2) -- both so the - # concurrency group below dedupes correctly across the two trigger - # paths, and for consistency with the dispatch payload. The Resolve - # step still strips a leading "v" defensively if one sneaks in. + # inputs.version is expected bare (no leading "v"), matching the dispatch payload. version: description: "Deployed prod version, e.g. 6.111.0-prod.2 (bare, no leading v)" required: true @@ -86,17 +30,7 @@ on: type: boolean default: false -# A pod retry (the AnalysisTemplate Job has backoffLimit: 1) can re-fire the -# repository_dispatch for the same version, and a manual workflow_dispatch -# re-run could race a real dispatch for that same version. Serialize on -# version so duplicate runs queue instead of double-posting to Slack. -# -# This group key is built from the RAW trigger value, before the Resolve -# step below can strip a leading "v" -- GitHub Actions expressions have no -# string-replace function, so normalization has to happen at the source -# instead (rollout-complete.yaml strips "v" in the dispatch payload) for -# repository_dispatch and workflow_dispatch to actually dedupe against each -# other for the same release. +# Serializes on version so a retry or manual re-run doesn't double-post to Slack. concurrency: group: prod-release-notes-${{ github.event.client_payload.version || inputs.version }} cancel-in-progress: false @@ -112,15 +46,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - # shipped_stories.py resolves the shipped commit range from prod/* - # git tags, so the checkout needs the tags themselves, not just full - # commit history. + # Needed for shipped_stories.py's prod/* tag resolution. fetch-tags: true - # Moved up front (both LLM steps in this workflow need the `claude` - # CLI): the release-notes prose step below, AND the opt-in triage - # explainer, which now runs earlier in the pipeline than prose does. - # One install, shared by both, instead of two. + # Shared by both the release-notes and triage-explainer Claude Code steps. - name: Set up Node uses: actions/setup-node@v4 with: @@ -145,11 +74,7 @@ jobs: CHART_VERSION="$DISPATCH_CHART_VERSION" fi - # rollout-complete.yaml's `version` arg is expected bare (it strips - # a leading "v" before building the dispatch payload -- see the - # concurrency.group comment above), but this strip stays as a - # defensive fallback in case a leading "v" ever sneaks in (e.g. a - # manual workflow_dispatch typo). + # Defensive fallback in case a leading "v" sneaks in. VERSION="${RAW_VERSION#v}" if [[ -z "$VERSION" ]]; then @@ -177,12 +102,7 @@ jobs: - name: Mark shipped stories as deployed id: mark_deployed - # A Shortcut bookkeeping failure (missing token, API error, or the - # silent-no-op guard's sys.exit(2)) must never skip release-notes - # generation and Slack posting for an otherwise healthy rollout -- - # moving stories to Done and announcing the release are independent - # concerns. continue-on-error surfaces the failure via - # steps.mark_deployed.outcome below instead of failing the job. + # A bookkeeping failure must never block release-notes generation and Slack posting. continue-on-error: true env: SHORTCUT_API_TOKEN: ${{ secrets.SHORTCUT_API_TOKEN }} @@ -197,27 +117,7 @@ jobs: - name: Reconcile Deploy Ready backlog id: reconcile - # Separate concern from this release's own commit-range bookkeeping - # above: this sweeps EVERY non-archived Deploy Ready story org-wide - # and transitions any whose linked PR already reached prod, - # regardless of which release shipped it (RC2 — see - # reconcile_deploy_ready.py's own docstring for the four guards it - # applies before trusting a linked PR). Its report file is never - # read by the release-notes/Slack steps below — see the workflow - # header comment for why that separation is load-bearing. It's - # written to $RUNNER_TEMP rather than the checkout (GITHUB_WORKSPACE) - # specifically so it can't leak into the prose step: that step's - # prompt only NAMES shipped-stories.json, but the agent still holds - # Glob+Read over its whole working directory, and a same-directory - # JSON full of real story names is exactly the "old stories leak - # into today's announcement" failure this whole separation exists to - # prevent — a distinct filename is a naming convention, not an - # access boundary. A failure here (Shortcut API hiccup, gh - # unreachable, a transition error) must never skip release-notes - # generation and Slack posting for an otherwise healthy rollout, so - # this gets the same continue-on-error treatment as "Mark shipped - # stories as deployed" above, and the two outcomes are - # warned/alerted together by the two steps that follow. + # Report goes to $RUNNER_TEMP, not the checkout, so it can't leak into the prose step below. continue-on-error: true env: SHORTCUT_API_TOKEN: ${{ secrets.SHORTCUT_API_TOKEN }} @@ -265,24 +165,7 @@ jobs: - name: Merge current-release backfill into shipped-stories.json id: merge_backfill - # Runs AFTER "Reconcile Deploy Ready backlog" writes its report, - # and BEFORE "Generate release notes" reads shipped-stories.json -- - # order is load-bearing here. Folds in the ONE, already-decided - # exception to "the reconcile report never reaches the prose step": - # a story that sweep backfilled whose derived TRUE shipping release - # (git evidence, resolved in reconcile_deploy_ready.py itself) is - # THIS run's own current release, not an earlier one. See - # merge_release_backfill.py's own docstring for why trusting that - # upstream decision, rather than re-deciding anything here, is what - # keeps this a pure mechanical merge. Reads the reconcile report - # from $RUNNER_TEMP (never the checkout) and only ever WRITES - # shipped-stories.json -- the report itself is still never handed - # to, or made readable by, the prose step below. - # - # continue-on-error: a failure here means, at worst, one backfilled - # story misses today's announcement (an omission, not a leak) -- - # never worth blocking release-notes generation and Slack posting - # for an otherwise healthy rollout, same posture as steps 2-3 above. + # Must run after "Reconcile Deploy Ready backlog" and before "Generate release notes". continue-on-error: true run: | REPORT_FILE="$RUNNER_TEMP/reconcile-deploy-ready-report.json" @@ -299,18 +182,9 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} EXPLAIN_TRIAGE_INPUT: ${{ inputs.explain_triage }} - # A repo-level Actions VARIABLE (Settings -> Secrets and - # variables -> Actions -> Variables), not a secret -- this is a - # plain on/off switch, nothing sensitive. Lets the automatic - # repository_dispatch trigger opt in too, since that trigger - # carries no workflow_dispatch-style inputs at all. + # Repo-level Actions variable, not a secret. ENABLE_TRIAGE_EXPLAINER_VAR: ${{ vars.ENABLE_TRIAGE_EXPLAINER }} run: | - # The actual decision rule (triage_explainer.resolve_enabled) is - # tested Python, not bash string comparisons duplicated here -- - # see build/ci/triage_explainer.py's own docstring. Off by - # default on every trigger path until a human explicitly flips - # one of the two switches. ENABLED=$(python3 build/ci/triage_explainer.py resolve-enabled \ --event-name "$EVENT_NAME" \ --explain-triage-input "$EXPLAIN_TRIAGE_INPUT" \ @@ -320,18 +194,9 @@ jobs: - name: Explain Deploy Ready triage backlog (optional) id: explain_triage if: steps.triage_opt_in.outputs.enabled == 'true' - # Opt-in and best-effort: a triage explanation is a convenience for - # whoever works the backlog next, never a precondition for - # anything downstream. continue-on-error keeps a failure here from - # ever touching the release announcement -- same posture as "Mark - # shipped stories as deployed" / "Reconcile Deploy Ready backlog" - # above. + # Opt-in and best-effort; never blocks the release announcement. continue-on-error: true env: - # This IS `ANTHROPIC_API_KEY` (correctly spelled) -- verified - # against this repo's actual configured secrets (`gh secret - # list`) rather than assumed, and it's the exact same secret the - # "Generate release notes" step below already uses. ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: | if [[ -z "$ANTHROPIC_API_KEY" ]]; then @@ -345,13 +210,6 @@ jobs: exit 0 fi - # Structural isolation, not a prompt instruction: this extracts - # ONLY the triage bucket (plus prod_tag) from the reconcile - # report into its own file -- "shipped" and "pending" are never - # even present in what the model reads, so there is nothing - # there for a prompt-injection path (a story's own - # description/comments are contributor-controlled text) to leak - # or influence. See triage_explainer.py's own docstring. TRIAGE_ONLY_FILE="$RUNNER_TEMP/reconcile-triage-only.json" python3 build/ci/triage_explainer.py extract --report "$REPORT_FILE" --out "$TRIAGE_ONLY_FILE" @@ -362,16 +220,8 @@ jobs: fi ANNOTATED_FILE="$RUNNER_TEMP/reconcile-triage-annotated.json" - # --allowedTools instead of --dangerously-skip-permissions: same - # reasoning as "Generate release notes" below -- the input embeds - # contributor-controlled text (story descriptions, comments), a - # prompt-injection path. This agent never holds Bash or network - # tools, and it never touches Shortcut or git itself: it only - # reads the triage-only JSON and writes prose hypotheses to - # $RUNNER_TEMP, which the prose step further below never reads -- - # same "not the checkout, not a naming convention, an actual - # access boundary" reasoning as the reconcile report itself (see - # the workflow header and reconcile_deploy_ready.py's docstring). + # --allowedTools, not --dangerously-skip-permissions: the input embeds + # contributor-controlled text, a prompt-injection path. No Bash/network tools. claude -p "Read the JSON file at ${TRIAGE_ONLY_FILE}. It lists Shortcut stories an automated Deploy Ready reconciliation sweep could not classify as shipped or pending ('triage'), each with: name, description, comments, and (for a story with a linked PR that didn't qualify) linked_prs -- the PR number and exactly which shipping-evidence guard(s) it failed (not merged / wrong repo / wrong target branch), or (for a story on the wrong Shortcut workflow) its workflow_id/workflow_state_id. For EACH story in the 'triage' array, propose (a) a short hypothesis for why it is stuck, grounded ONLY in the fields given -- e.g. 'a linked PR targets preprod, not master -- likely a promotion PR was linked instead of the real feature PR', 'no PR is linked at all -- may have shipped via an unlinked PR, or a PR may still need to be opened', 'workflow/state ids do not match Standard -- this story may belong to a different team or process entirely', and (b) a short suggested next action for a human, e.g. 'search for the real feature PR and re-link it', 'no code artifact found -- consider closing manually', 'may belong to another repo's release train -- verify with that PR's author'. Do not invent PR numbers, dates, names, or any fact not present in the input. Do not recommend or imply any automatic action, transition, or comment -- every output is an unverified hypothesis for a human to check, never a finding. Prefix every hypothesis string with the literal text '[AI hypothesis, unverified] ' so it can never be mistaken for a verified cause. Write your ONLY output as JSON to ${ANNOTATED_FILE}: a list of objects, one per input triage story, each shaped {id, hypothesis, suggested_next_action}. Do not ask any clarifying questions -- proceed directly." \ --allowedTools "Read,Write,Glob,Grep" @@ -394,12 +244,8 @@ jobs: fi mkdir -p release-notes-output - # --allowedTools instead of --dangerously-skip-permissions: the - # input (shipped-stories.json) embeds contributor-controlled text - # (commit subjects, branch names, Shortcut free text), which is a - # prompt-injection path. This agent must never hold Bash or - # network tools, so the fix is least-privilege tool scoping at the - # provisioning layer, not content filtering. + # --allowedTools, not --dangerously-skip-permissions: shipped-stories.json + # embeds contributor-controlled text, a prompt-injection path. No Bash/network tools. claude -p "Use the sefaria-release-notes skill (.claude/skills/sefaria-release-notes/SKILL.md). Generate release notes from the shipped-stories JSON file at shipped-stories.json. Output directory: release-notes-output/. Do not ask any clarifying questions — proceed with defaults documented in the skill's Edge Cases table." \ --allowedTools "Read,Write,Glob,Grep" @@ -421,11 +267,7 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_PRODUCT_WEBHOOK }} run: | - # Guarded in the script body, not in `if:` — a step's own `env` block - # is not reliably visible to that same step's `if` expression, so - # `env.SLACK_PRODUCT_WEBHOOK != ''` there silently evaluates empty and - # the step never runs. Until the second webhook exists this is a - # deliberate no-op that must not fail the workflow. + # Guarded here, not in `if:` — a step's own `env` isn't reliably visible to its own `if`. if [[ -z "$SLACK_WEBHOOK_URL" ]]; then echo "SLACK_PRODUCT_WEBHOOK is not set — skipping the release announcement post." exit 0 diff --git a/build/ci/README-prod-release-notes.md b/build/ci/README-prod-release-notes.md index f8e5dce696..7e4a759e53 100644 --- a/build/ci/README-prod-release-notes.md +++ b/build/ci/README-prod-release-notes.md @@ -1,458 +1,89 @@ # Prod rollout → Slack + release notes: manual setup -This repo's changes (below) are necessary but not sufficient — the following -still needs a human with real credentials, since none of it can be -generated or guessed by an agent. +This repo's changes are necessary but not sufficient — some setup below +requires a human with real credentials. -## How it works +## Pipeline ``` Argo post-promotion analysis (prod) -> repository_dispatch (prod-rollout-succeeded, carries `version` + `chartVersion`) - -> build/ci/shipped_stories.py — walks the prod/* tag range in git, - resolves Shortcut story codes from - commit subjects, merged-PR branch - names, AND (as a third, fallback - source) Shortcut's own PR<->story - link, hydrates story details - -> build/ci/mark_stories_deployed.py — moves each shipped story - Deploy Ready -> Done via the - Shortcut API. Only ever writes that - state — no comment, no other - annotation. A failure here (missing - token, API error, nothing to move) is - logged and Slack-alerted but never - blocks the steps below. - -> build/ci/reconcile_deploy_ready.py — separately, sweeps EVERY - non-archived Deploy Ready story - org-wide (not just this release's - commit range) and transitions any - whose linked PR already reached prod - — see "Reconciliation sweep" below. - Its report is written to - `$RUNNER_TEMP` and NEVER read - wholesale by the steps that follow. - -> build/ci/merge_release_backfill.py — folds the ONE, pre-filtered - exception into shipped-stories.json: - a story the sweep backfilled whose - derived TRUE shipping release is - THIS run's own current release, not - an earlier one — see "Backfilled - stories that shipped in THIS - release" below. Only ever touches - shipped-stories.json; never reads - the reconcile report's shipped/ - pending/triage buckets wholesale. - -> build/ci/triage_explainer.py — OPT-IN, off by default: extracts - + headless `claude -p` ONLY the triage bucket into its own - file, then proposes a labeled - hypothesis + suggested next action - per triage story — see "Opt-in triage - explainer" below. Never mutates - Shortcut; its output NEVER reaches - the two steps that follow either. - -> sefaria-release-notes skill — reads shipped-stories.json (now - including the merge step's backfill, - if any), writes prose only - -> scripts/post_to_slack.py — posts both files to Slack + -> build/ci/shipped_stories.py — resolves shipped Shortcut stories for the release + -> build/ci/mark_stories_deployed.py — moves those stories Deploy Ready -> Done + -> build/ci/reconcile_deploy_ready.py — org-wide Deploy Ready sweep (independent of this release) + -> build/ci/merge_release_backfill.py — folds any current-release backfill into shipped-stories.json + -> build/ci/triage_explainer.py — opt-in: proposes hypotheses for the reconcile sweep's triage bucket + + headless `claude -p` + -> sefaria-release-notes skill — reads shipped-stories.json, writes prose only + -> scripts/post_to_slack.py — posts both files to Slack ``` -Only the release-notes generation step is an LLM. Deciding which stories a given deploy closed -is a graph walk over git history, Shortcut IDs and the Shortcut API, and -flipping a story's workflow state is a for-loop over a REST API — none of -that is a job for a model. The skill's only input is the JSON that -`shipped_stories.py` already produced; it does not call GitHub or Shortcut -itself, and it does not mutate any story. +Only the release-notes prose step and the opt-in triage explainer are LLM +steps; everything else is deterministic Python or a REST call. -## Three discovery sources in `shipped_stories.py` +## Running the scripts -For each commit in the resolved tag range, a story id is looked for in, in -order: - -1. **The commit subject itself** (`sc-NNNNN` in any of its usual shapes — - `fix(sc-123):`, `[sc-123]`, `feature/sc-123`, ...). -2. **The branch name of the commit's merged PR**, when the commit carries a - `(#N)` reference or is a bare "Merge pull request #N from ..." — some - teams put the story code in the branch instead of the commit message. -3. **Shortcut's own PR<->story link** (`GET search/stories?query=pr:`), - used ONLY as a fallback for a commit whose PR carries no story id from - either source above. This exists because git text is not the only place - a story/PR link can live — a story can be attached to a PR from the - Shortcut UI with no story code ever appearing in the branch name. - `branch:"..."` and `pull-request:N` do NOT resolve this on this org; - only the `pr:N` search operator does. The id is adopted ONLY when the - search returns EXACTLY one story — an ambiguous match (>1) is logged and - skipped rather than guessed. - - **This fallback is guarded the same way `reconcile_deploy_ready.py`'s - sweep is (via the shared `shortcut_pr_guards.py`), and for the same - reason: a bare `pr:` match only proves Shortcut linked SOME story to - that PR number, not that the PR is real shipping evidence.** Verified - live: a promotion PR (head branch `master`/`preprod`/`prod`, merging - into the next environment) resolves via `pr:` to a real story just as - readily as that story's actual feature PR does, while proving nothing - about whether that story's own change shipped. So the fallback (a) is - never even attempted for a commit whose subject is auto-generated - merge/branch-sync noise (`NOISE_PATTERN` — the same pattern already used - to keep such commits out of `commits_without_story`) or whose PR's own - head branch is a long-lived environment branch, and (b) re-checks the - single search result's own linked-PR entry against the four PR-level - guards (merged / Sefaria-Project repo / target branch master / head - branch not a long-lived environment branch) before adopting it — a - match that fails those guards is a warn-and-skip. - - Ids recovered this way are echoed separately in the output's - `stories_from_shortcut_pr_link` list (in - addition to the ordinary `story_ids`) so a report can call out what only - Shortcut knew. Gated on `SHORTCUT_API_TOKEN`; without it (or on any - per-lookup failure) this step is skipped/warned and the run continues - with git-only discovery — it never aborts `shipped_stories.py`. - -## Reconciliation sweep: `build/ci/reconcile_deploy_ready.py` - -`shipped_stories.py` + `mark_stories_deployed.py` only ever look at ONE -release's commit range (`prev-tag..cur-tag`). A story whose PR merged and -shipped in an EARLIER release — or before this pipeline existed — never -gets revisited by that pair of scripts; nothing ever walks backward and -re-checks a story sitting in Deploy Ready. Of the stories stuck in Deploy -Ready when this was diagnosed, ten times as many were this class of gap as -were the discovery gap `shipped_stories.py`'s third source fixes above. - -`reconcile_deploy_ready.py` is a standalone, org-wide sweep that closes -that gap. It enumerates every non-archived Deploy Ready story, resolves -each one's linked merged PR(s), and checks whether any of those PRs' -merge commits are an ancestor of the current prod tag. It classifies every -story into exactly one of three buckets: - -- **shipped** — at least one qualifying PR is in prod. Transitioned - Deploy Ready (500000045) -> Done (500000010). -- **pending** — has a qualifying merged PR, but none are in prod yet. Left - alone — this is the correct state, not a bug. -- **triage** — no qualifying PR at all, or the story lives in a - non-Standard Shortcut workflow. Left alone and reported; this is the - part of the output a human actually has to look at. - -### The five guards - -Each of these caught a real false positive while this script was built (or -maintained) against live data — skipping any one of them silently -mis-transitions a story. The first four (merged / repo / target branch / -head branch) are PR-level checks shared with `shipped_stories.py`'s own -RC1 PR-link fallback via `build/ci/shortcut_pr_guards.py` — both scripts -ask the same underlying question ("does this linked PR actually prove a -story's change reached prod?") and a promotion PR is exactly as good at -fooling either one, so there is exactly one implementation of these four -checks, not two parallel copies that could silently drift apart. They -already have, twice, in OPPOSITE directions: - -1. **`repository_id` must be Sefaria-Project's (`500000103`).** A story can - link a PR from a different repo; resolving that PR number against - Sefaria-Project instead finds an unrelated (often much older) PR that - happens to share the number — and that PR can easily already be in - prod, which would report "shipped" for a story that never touched this - repo. -2. **`target_branch_name` must be `"master"`.** Some stories link a - promotion PR (preprod -> prod, or master -> preprod) instead of, or - alongside, the actual feature PR. A promotion PR merges constantly and - proves nothing about whether this story's own change reached prod. -3. **The PR's own HEAD (source) branch must NOT be a long-lived - environment branch** (`master`, `preprod`, `prod`). Verified live: a - promotion PR merging `preprod` INTO `master` passes guards 1, 2 and 4 - cleanly — it's merged, against the right repo, and its target really is - `master` — yet it's still a promotion merge, not a feature PR. Guard 2 - alone cannot catch this shape (a promotion merge legitimately targets - `master`); only the HEAD branch gives it away. **This is the guard that - was missing** when a real story was classified `shipped` on the - strength of a PR shaped exactly this way, while its genuine feature PR - was (correctly) rejected by guard 2 for targeting a hotfix branch - instead of `master` directly. `shipped_stories.py`'s RC1 fallback - already had this check for its own purposes before `reconcile_ - deploy_ready.py` did — the two scripts drifted apart on whether this - guard existed at all before it was unified here. -4. **`merged` must be `true`.** An open or closed-without-merging PR is not - evidence anything shipped. -5. **`workflow_id` must be the Standard workflow (`500000005`), and - `workflow_state_id` must be exactly the numeric Deploy Ready id - (`500000045`).** The Shortcut state named "Deploy Ready" — note its real - name carries a trailing space, `"Deploy Ready "` — is workflow-specific: - `500000045` doesn't exist as a concept in, say, the Content workflow. - Enumeration is keyed on the state NAME (the search endpoint has no other - way to filter it), so classification re-checks the NUMERIC ids before - trusting a match; a story on any other workflow, or at any other state - id despite matching the name, is routed to triage with its actual - workflow/state ids reported — mirroring `mark_stories_deployed.py`'s - `skipped_different_workflow` handling. This one is story-level, not - PR-level, and stays local to `reconcile_deploy_ready.py`. - -Enumeration uses the token'd search endpoint -(`search/stories?query=state:"Deploy Ready" !is:archived`), paginated via -its `next` cursor. This is deliberately NOT `iterations-get-active` — that -endpoint is silently scoped to the calling token's own teams and has -already produced an incomplete picture for this team once; the search -endpoint returns every matching story across every team. - -A qualifying PR's merge commit is resolved via `gh pr view --json -mergeCommit` and tested with `git merge-base --is-ancestor ` — verified to correctly discriminate a merged-but-not-yet-promoted PR -from one that already reached prod. `git log --grep="(#N)"` was tried and -rejected: it misses squash-merge subjects and can't tell a real promotion -merge apart from an unrelated one. - -### Dry-run by default +``` +python3 build/ci/shipped_stories.py --version 6.111.0-prod.2 [--out shipped-stories.json] [--repo Sefaria/Sefaria-Project] [--chart-version 0.87.5-prod.1] +python3 build/ci/shipped_stories.py --range .. [--out shipped-stories.json] +``` +Requires `git` and `gh` on PATH. `SHORTCUT_API_TOKEN` is optional; without +it, story ids are still emitted but hydration and the PR-link fallback are +skipped. -Unlike `mark_stories_deployed.py` (which mutates by default and needs -`--dry-run` to preview), `reconcile_deploy_ready.py` inverts that: **it -never mutates anything unless you pass `--apply`.** This is a bulk mutation -of shared state across potentially many stories and several different -teams, and — unlike a single release's handful of stories — there's no -natural moment (a deploy just happened) that makes running it low-risk. An -explicit `--dry-run` flag also exists, purely for symmetry with -`mark_stories_deployed.py` and CI readability; it's a no-op since dry-run -is already the default, and it always wins if both flags are passed -together. +``` +python3 build/ci/mark_stories_deployed.py --input shipped-stories.json [--dry-run] \ + [--workflow-id 500000005] [--from-state-id 500000045] [--done-state-id 500000010] +``` +Requires `SHORTCUT_API_TOKEN` unless `--dry-run` is passed. ``` -python3 build/ci/reconcile_deploy_ready.py --dry-run # classify + report, mutate nothing (default) -python3 build/ci/reconcile_deploy_ready.py --apply # actually transition the "shipped" bucket +python3 build/ci/reconcile_deploy_ready.py [--dry-run] +python3 build/ci/reconcile_deploy_ready.py --apply python3 build/ci/reconcile_deploy_ready.py --apply --prod-tag prod/6.111.0-prod.2+chart.0.87.5-prod.1 --out report.json ``` - -**Critical: this script never reads or writes `shipped-stories.json` itself -and never talks to the release-notes prose step directly.** Most of what it -backfills shipped in EARLIER releases — leaking that wholesale into today's -release announcement would have Slack claim a dozen old features shipped -today. This script only ever transitions Shortcut state; it never posts a -comment or any other annotation ("just mark it as done" is the whole job -here). In the workflow, its step runs after `mark_stories_deployed.py` and -writes its own separate report file to `$RUNNER_TEMP` (NOT the checkout / -`GITHUB_WORKSPACE`) — a distinct filename alone is a naming convention, not -an access boundary, and the release-notes step's headless Claude run holds -`Glob`+`Read` over its whole working directory, so a same-directory JSON -full of real, recently-shipped story names would be one bad glob away from -leaking into the prose it writes. Keeping the report outside the checkout -entirely is what actually enforces the separation. A failure here is -warned/Slack-alerted the same way a `mark_stories_deployed.py` failure is, -and never blocks release-notes generation or posting. - -One narrow, explicitly-filtered exception to "never feeds the prose step" -exists — see the next section. - -## Backfilled stories that shipped in THIS release: `build/ci/merge_release_backfill.py` - -The reconciliation sweep exists because nothing else revisits a Deploy -Ready story once it falls outside the current release's git-range scan. -Most of what it finds shipped in an EARLIER release, and the previous -section is about keeping THAT out of today's announcement. But some of -what it finds shipped in the CURRENT release too — a race, a discovery gap -`shipped_stories.py`'s own RC1 fallback didn't close, a story that simply -never got picked up by `mark_stories_deployed.py`'s own commit-range scan. -Silently excluding that story from today's announcement is a different -flavor of the same underlying mistake ("say what actually shipped today"), -just by omission instead of leakage. - -Every entry in `reconcile_deploy_ready.py`'s `shipped` bucket now carries a -`shipping_release_tag` field: the TRUE release that shipped it, resolved -via `resolve_shipping_release_tag()` (`git tag --list 'prod/*' --contains - --sort=creatordate | head -1` — the first, earliest-created, -`prod/*` tag that actually contains the winning PR's merge commit). Note -the ascending `--sort=creatordate` — the OPPOSITE of the default-prod-tag -resolution elsewhere in this script, which wants the NEWEST tag; this one -wants the OLDEST tag that still contains the commit, i.e. the first -release it ever reached. When `shipping_release_tag` equals the CURRENT -prod tag that run was checking against, the entry ALSO carries a -`hydrated_story` sub-object — a story record in exactly -`shipped_stories.py`'s own hydrated-story shape (`id`, `name`, -`description`, `url`, `workflow_id`, `workflow_state_id`, `story_type`), -built from data the sweep already had in memory (no extra Shortcut API -call). When `resolve_shipping_release_tag()` can't determine a release at -all (shallow checkout, a genuine gap in tag history), the entry gets -NEITHER field populated with a guess — fail closed, never guess a story -into an announcement. - -`merge_release_backfill.py` is a separate, tiny, deterministic script that -trusts that upstream decision completely and makes no judgment of its own: -it reads `reconcile_deploy_ready.py`'s report, takes ONLY the -`hydrated_story` entries (every other shipped entry — earlier release, -unresolvable release, or a pending/triage entry — carries no such field -and is silently skipped), and folds them into `shipped-stories.json`'s own -`story_ids`/`stories` lists, deduplicated by story id against what -`shipped_stories.py`'s own git-range + RC1 discovery already found (a -story both paths independently find is never duplicated; the pre-existing -entry always wins). The merged ids are also recorded separately under -`stories_from_reconciliation_backfill`, mirroring `shipped_stories.py`'s -own `stories_from_shortcut_pr_link` provenance field. +`--dry-run` is the default; nothing is transitioned without `--apply`. +Requires `git`, `gh`, and `SHORTCUT_API_TOKEN` (required even for `--dry-run`). ``` python3 build/ci/merge_release_backfill.py \ --shipped-stories-out shipped-stories.json \ --reconcile-report reconcile-deploy-ready-report.json - # writes the merged result back to --shipped-stories-out by default; - # pass --out to write somewhere else instead. + # writes the merged result back to --shipped-stories-out by default; pass --out to write elsewhere ``` -In the workflow, this step runs immediately after `reconcile_deploy_ready.py` -and BEFORE the release-notes prose step — order is load-bearing, since the -prose step reads `shipped-stories.json` once and needs the merge already -done. It reads the reconcile report from `$RUNNER_TEMP` (never the -checkout) and only ever WRITES `shipped-stories.json` — the report's -`shipped`/`pending`/`triage` buckets as a whole are still never handed to, -or made readable by, the prose step; only the pre-filtered -`hydrated_story` entries this script extracts ever reach it. A failure -here means, at worst, one backfilled story misses today's announcement (an -omission, not a leak) — never worth blocking release-notes generation and -Slack posting over, so this step is `continue-on-error` too. - -## Opt-in triage explainer - -`reconcile_deploy_ready.py`'s triage bucket is reported as -`reason=no_qualifying_pr` (or `non_standard_workflow_or_state`) plus raw -diagnostic fields — description, comment text, and, per linked PR, exactly -which shipping-evidence guard it failed (see `_triage_context` / -`_diagnose_linked_pr` in `reconcile_deploy_ready.py`). That still leaves a -human to open every triage story and work out each one individually. An -OPT-IN workflow step proposes a short, labeled hypothesis for each one -instead — but the pattern is deliberately the same one this workflow -already uses for release-notes prose, kept LLM-free everywhere it can be: - ``` -reconcile_deploy_ready.py — writes its report (unchanged; still - deterministic, still stdlib-only, - still no API client) - -> build/ci/triage_explainer.py — extracts ONLY the triage bucket - "extract" subcommand (+ prod_tag) into its own file - -> headless `claude -p` — reads THAT file, writes a - (in the workflow step only) {id, hypothesis, suggested_next_action} - list to a separate file +python3 build/ci/triage_explainer.py extract --report reconcile-deploy-ready-report.json --out triage-only.json +python3 build/ci/triage_explainer.py resolve-enabled --event-name workflow_dispatch --explain-triage-input true --enable-var "" ``` -**Why a separate file, not a prompt instruction to "only look at -triage":** the full report's `shipped`/`pending` buckets must never be -visible to, scored by, or able to influence this explainer, and a triage -story's `description`/`comments` are CONTRIBUTOR-CONTROLLED TEXT — a -prompt-injection path. Rather than trust the model to honor "ignore the -other buckets" against adversarial input embedded in the very document -it's reading, `triage_explainer.py extract` simply never puts -shipped/pending data into the file the explainer is given at all. There is -nothing there to leak or be steered by, structurally, not merely by -convention — see that script's own docstring, and -`build/ci/tests/test_triage_explainer.py`, which asserts the extracted -document never contains shipped/pending data even when the source report -does. - -**Everything else about this step mirrors "Generate release notes" -below**, on purpose: - -- `--allowedTools "Read,Write,Glob,Grep"`, never - `--dangerously-skip-permissions` — no Bash, no network. This agent - cannot touch Shortcut, git, or the GitHub API even if it wanted to; - writing English from a file it's handed is the entire extent of what it - can do. -- Its output (`$RUNNER_TEMP/reconcile-triage-annotated.json`) stays in - `$RUNNER_TEMP`, never the checkout — same reasoning as the reconcile - report itself (see "Reconciliation sweep" above): the release-notes - prose step holds `Glob`+`Read` over its whole working directory, and a - distinct filename is a naming convention, not an access boundary. -- Every hypothesis string is required (by the prompt) to start with the - literal `[AI hypothesis, unverified]` marker, so it can never be - mistaken for a verified finding by whoever reads it. -- **Proposes, never decides**: no Shortcut mutation, no transition — of - ANY kind, on ANY story. Authority stays entirely with the deterministic - layer (the five guards, the ancestry check) and the human reading the - report. An ambiguous `pr:` - lookup returning more than one story is still a deterministic - warn-and-skip in `shipped_stories.py`/`reconcile_deploy_ready.py` - (unchanged) — that's precisely the "model decides two things are - related and closes the wrong one" failure this design rejects, and the - explainer never gets a vote on it either. - -**Opt-in, off by default on every trigger path.** A `workflow_dispatch` -run opts in per-run via its own `explain_triage` input; the automatic -`repository_dispatch` trigger carries no such input at all (it's not a -`workflow_dispatch`), so it instead opts in via a repo-level Actions -*variable* (`vars.ENABLE_TRIAGE_EXPLAINER`, Settings → Secrets and -variables → Actions → **Variables**, not Secrets — it's a plain on/off -switch, nothing sensitive). The actual decision rule -(`triage_explainer.resolve_enabled`) is tested Python, not a bash string -comparison duplicated inline in the workflow — see -`build/ci/tests/test_triage_explainer.py`. It degrades cleanly and never -blocks the release announcement on any of these paths: - -- **Disabled** (the default): the step's `if:` condition is false; it - never runs at all. -- **No `ANTHROPIC_API_KEY`**: the step's own guard exits 0 immediately — - same posture as "Generate release notes" below, which requires the key - (this step is opt-in, so it degrades instead of failing the job). -- **No triage stories this run**: skipped with a short message — nothing - to explain. -- **The `claude -p` call itself fails**: `continue-on-error: true` (same - as "Mark shipped stories as deployed" / "Reconcile Deploy Ready - backlog") — a failure here is never allowed to fail the job or skip - release-notes generation and Slack posting. - -**A note on the API key secret name:** an earlier version of this -instruction claimed the repo's secret is misspelled `ANTHOPIC_API_KEY` -(no R) and that the workflow maps it deliberately. That claim was checked -against this repo's actual configured secrets (`gh secret list`) before -writing any code — the only secret that exists is the correctly-spelled -`ANTHROPIC_API_KEY`, already used by both `manual-promotion.yaml` and this -workflow's own "Generate release notes" step. This step uses that same, -correctly-spelled secret; wiring in the claimed misspelling would have -referenced a secret that doesn't exist, silently and permanently -disabling this feature in production (`secrets.ANTHOPIC_API_KEY` always -resolves to an empty string, which the step's own missing-key guard would -treat as "no key" on every single run). - ## What's already wired up in this repo - `helm-chart/sefaria/templates/analysistemplate/rollout-complete.yaml` — - a `notify-github` container, gated on `deployEnv == "production"` (the - prod HelmRelease sets `deployEnv: production`, not `"prod"`), fires a - `repository_dispatch` (`event_type: prod-rollout-succeeded`) once Argo's - post-promotion analysis confirms the rollout healthy. Reads a - `GH_DISPATCH_TOKEN` key from its OWN dedicated secret - (`.Values.secrets.ghDispatch.ref`, default `gh-dispatch-token`) — kept - separate from `local-settings-secrets` because that secret is mounted via - `envFrom` into every web/task/monitor/cronjob pod, which is far too broad - a blast radius for a GitHub PAT. + fires `repository_dispatch` (`prod-rollout-succeeded`) once Argo's + post-promotion analysis confirms the prod rollout healthy. Uses a + `GH_DISPATCH_TOKEN` from its own dedicated secret + (`.Values.secrets.ghDispatch.ref`, default `gh-dispatch-token`). - `.github/workflows/prod-release-notes.yaml` — listens for that dispatch - (or a manual `workflow_dispatch`), resolves the version (and optional - chart version, for disambiguating a chart-only rollout), runs - `shipped_stories.py` and `mark_stories_deployed.py`, separately runs - `reconcile_deploy_ready.py` (its report never reaches the steps below), - runs the `sefaria-release-notes` skill headlessly, and posts both output - files to Slack via `scripts/post_to_slack.py`. The reconcile step honors - the same `workflow_dispatch` `dry_run` input as `mark_stories_deployed.py` - does — real trigger or `dry_run=false` passes `--apply`; the default - `workflow_dispatch` (`dry_run=true`) leaves it in its default dry-run - mode. -- `.claude/skills/sefaria-release-notes/` — the skill, shipped in-repo, - now takes a shipped-stories JSON file as its only input and only writes - prose. It no longer talks to GitHub or Shortcut. -- **preprod needs no changes.** `rollout-complete-preprod` already exists - (same chart, templated per `deployEnv`) and already posts to Slack on a - successful preprod rollout — assuming its `SLACK_URL` is populated (see - below). + (or a manual `workflow_dispatch`), runs the pipeline above, and posts to + Slack. +- `.claude/skills/sefaria-release-notes/` — the release-notes skill, + shipped in-repo, takes a shipped-stories JSON file as its only input. +- Preprod needs no changes — `rollout-complete-preprod` already exists and + posts to Slack on a successful preprod rollout (as long as `SLACK_URL` + is populated; see below). ## Still required — infrastructure repo (SOPS-encrypted secret) -**This is the step that makes the whole pipeline live — without it, -`GH_DISPATCH_TOKEN` is simply absent, the dispatch curl gets a 401, falls -through `|| /bin/true`, and the entire feature is a silent no-op with -nothing failing anywhere.** - 1. Create a GitHub PAT scoped to `Sefaria/Sefaria-Project` only — - fine-grained, **Contents: read and write** permission (required for the - `repository_dispatch` API endpoint; this token never needs push/admin - access, it only fires a dispatch event). -2. SOPS-encrypt it into the `infrastructure` repo as its OWN dedicated - Secret (NOT `local-settings-secrets` — that secret is mounted into every - pod in the deployment; see above) under key `GH_DISPATCH_TOKEN`. This - repo's `envs/prod/helmrelease.yaml` already points - `secrets.ghDispatch.ref` at `gh-dispatch-token-production`; the - infrastructure repo needs to create a Secret with that exact name. -3. Confirm `flux reconcile` picks it up (or wait for the next 5-minute - poll) so the key exists on the `rollout-complete-production` Job's pod - before the next prod rollout. + fine-grained, **Contents: read and write** permission. +2. SOPS-encrypt it into the `infrastructure` repo as its own dedicated + Secret under key `GH_DISPATCH_TOKEN`, named `gh-dispatch-token-production` + to match `envs/prod/helmrelease.yaml`'s `secrets.ghDispatch.ref`. +3. Confirm `flux reconcile` picks it up before the next prod rollout. + +Without this, `GH_DISPATCH_TOKEN` is absent, the dispatch curl 401s and +falls through `|| /bin/true`, and the whole feature is a silent no-op. ## Still required — Sefaria-Project GitHub Actions secrets @@ -460,101 +91,55 @@ Add these under repo Settings → Secrets and variables → Actions: | Secret | Purpose | Notes | |---|---|---| -| `SHORTCUT_API_TOKEN` | `shipped_stories.py` story hydration and PR-link fallback, `mark_stories_deployed.py` state transitions, `reconcile_deploy_ready.py` enumeration and transitions | Shortcut → Settings → API Tokens. Not the same as the OAuth MCP connection used interactively. | -| `SLACK_PRODUCT_WEBHOOK` | Non-technical release announcement | A second Slack incoming webhook, pointed at whichever channel should get `release-announcement-product-slack.txt`. Until this is set, that post step is a guarded no-op (won't fail the workflow). | +| `SHORTCUT_API_TOKEN` | Story hydration, PR-link fallback, and state transitions for all three CI scripts | Shortcut → Settings → API Tokens | +| `SLACK_PRODUCT_WEBHOOK` | Non-technical release announcement | A second Slack incoming webhook. Until set, that post step is a guarded no-op | Already exist and are reused as-is: `SLACK_DEPLOY_WEBHOOK`, `GITHUB_TOKEN`, `ANTHROPIC_API_KEY`. -Optional, only if you want the triage explainer to opt in automatically on -the real `repository_dispatch` trigger (a manual `workflow_dispatch` run -can already opt in per-run via its own `explain_triage` input without -this): add a repo-level Actions **Variable** (Settings → Secrets and -variables → Actions → **Variables** tab, NOT Secrets) named -`ENABLE_TRIAGE_EXPLAINER` set to `true`. See "Opt-in triage explainer" -above. +Optional — to have the triage explainer opt in automatically on the real +`repository_dispatch` trigger (a manual `workflow_dispatch` run can already +opt in per-run via its `explain_triage` input): add a repo-level Actions +**Variable** (not Secret) named `ENABLE_TRIAGE_EXPLAINER` set to `true`. ## Worth verifying, not something this session could check `SLACK_URL` in `local-settings-secrets` — confirm it's actually populated -for **both** `preprod` and `prod` (not just present as a key). The existing -`rollout-complete` Slack ping silently no-ops if it's empty or missing -(`optional: true`), so a misconfigured value wouldn't surface as an error -anywhere — it would just be quiet. +for both `preprod` and `prod`. The existing `rollout-complete` Slack ping +silently no-ops if it's empty or missing (`optional: true`). -## End-to-end verification, once the above is done +## End-to-end verification -1. **Dry-run the whole pipeline without a real deploy.** Run: +1. Dry-run the whole pipeline without a real deploy: ``` gh workflow run "Prod Release Notes" -f version= -f dry_run=true ``` - Add `-f chart_version=` if that app version has more than - one `prod/*` tag (a chart-only rollout) and you need a specific one. - - This exercises tag-range resolution, story hydration, release-notes - generation, and both Slack posts, with `mark_stories_deployed.py` run in - `--dry-run` mode so nothing in Shortcut actually moves. It's the fastest - way to validate a change to any of the scripts or the skill without - waiting on a real rollout. - -2. **Then confirm the real trigger path.** Promote something small through - to prod normally. - -3. Watch for the existing terse Slack ping from `rollout-complete-production` - (confirms the AnalysisTemplate ran and Slack posting works at all). - -4. Watch the `Prod Release Notes` GitHub Actions workflow run - (`repository_dispatch` → `prod-rollout-succeeded`). If it doesn't fire, - check the `notify-github` container's logs on the `rollout-complete-production` - Job pod (`kubectl logs -n default -l job-name=...`) for the dispatch - curl's exit/response. The curl runs with `-sS -f --max-time 30 - --connect-timeout 10`, so a bad/missing token (HTTP 401/403) or a - timeout now prints to stderr in the pod logs — but the call still falls - through `|| /bin/true` by design so it never blocks or fails the - rollout, which means a failure here still won't surface anywhere except - those logs unless you go look. + Add `-f chart_version=` for a chart-only rollout. +2. Then confirm the real trigger path: promote something small through to + prod normally. +3. Watch for the existing terse Slack ping from `rollout-complete-production`. +4. Watch the `Prod Release Notes` GitHub Actions workflow run. If it + doesn't fire, check the `notify-github` container's logs on the + `rollout-complete-production` Job pod for the dispatch curl's + exit/response. 5. Confirm both Slack files post correctly, and confirm the shipped stories actually moved Deploy Ready → Done in Shortcut. ## Running the tests -The tests for these scripts (`build/ci/tests/test_shipped_stories.py`, -`test_mark_stories_deployed.py`, `test_reconcile_deploy_ready.py`, -`test_triage_explainer.py`, `test_merge_release_backfill.py`) are **not** -collected by the repo's root `pytest.ini` (that config is scoped to the -Django app's own test suites), so run them by explicit path from the repo -root — either the whole directory: - ``` python3 -m pytest build/ci/tests/ -q -p no:django -c /dev/null ``` -or each file explicitly: - -``` -python3 -m pytest build/ci/tests/test_shipped_stories.py build/ci/tests/test_mark_stories_deployed.py build/ci/tests/test_reconcile_deploy_ready.py build/ci/tests/test_triage_explainer.py build/ci/tests/test_merge_release_backfill.py -q -p no:django -c /dev/null -``` - -Both flags are needed even though only explicit file paths are passed: -`pytest` still discovers and loads the repo-root `pytest.ini` from the -current directory regardless of which paths are given on the command line, -and that ini sets `DJANGO_SETTINGS_MODULE` — which makes the `pytest-django` -plugin try to `django.setup()` the whole app (and fail with -`ModuleNotFoundError: No module named 'allauth'` in an environment that -hasn't installed the full Django app's dependencies, which these -standalone, stdlib-only scripts have no need of). `-c /dev/null` stops -`pytest.ini` from being read at all; `-p no:django` disables the -`pytest-django` plugin itself as a second, independent line of defense -(matters if some other ini/plugin-autouse path re-enables it). Depending on -which Python environment you invoke `pytest` from, the plain command -without these flags may happen to work (if that environment has the full -Django app's dependencies installed) or may not — the flagged command works -regardless. +Both `-c /dev/null` and `-p no:django` are needed: the repo-root +`pytest.ini` sets `DJANGO_SETTINGS_MODULE`, which makes `pytest-django` try +to `django.setup()` the whole app even when only these standalone, +stdlib-only scripts' tests are selected. `-c /dev/null` stops that ini from +being read; `-p no:django` disables the plugin as a second line of defense. No network access, `git`, or `gh` binary is required — everything that -would otherwise shell out or call the Shortcut API is monkeypatched at the -same boundary the module itself uses (`subprocess.run`, -`urllib.request.urlopen`, or the module's own `run_git`/helper functions). +would otherwise shell out or call the Shortcut API is monkeypatched in the +tests. diff --git a/build/ci/mark_stories_deployed.py b/build/ci/mark_stories_deployed.py index cdf329ee67..f414dcef77 100755 --- a/build/ci/mark_stories_deployed.py +++ b/build/ci/mark_stories_deployed.py @@ -1,60 +1,12 @@ #!/usr/bin/env python3 """ -Deterministically transition shipped Shortcut stories from "Deploy Ready" to "Done". - -Reads a shipped-stories JSON file (as produced by shipped_stories.py) and, for -each entry in its "stories" list whose workflow_id equals --workflow-id AND -workflow_state_id equals --from-state-id, PUTs a workflow_state_id update to ---done-state-id via the Shortcut API. Stories already at the done state are -skipped and logged. Stories sitting in any other state of --workflow-id are -skipped and logged. Stories belonging to a DIFFERENT workflow than ---workflow-id are skipped and logged separately, because --from-state-id and ---done-state-id are workflow-specific: the Sefaria org has roughly ten -Shortcut workflows, and a state id that means "Deploy Ready" in one workflow -can be meaningless (or mean something else entirely) in another. Silently -lumping those stories into "skipped, some other state" is exactly the -unattended-run failure mode this script exists to avoid — a shipped story in -a non-Standard workflow must never look indistinguishable from "nothing -needed moving". No story is ever mutated in any of these three skip cases. +Transitions shipped Shortcut stories from "Deploy Ready" to "Done". Usage: python3 mark_stories_deployed.py --input shipped-stories.json [--dry-run] \ [--workflow-id 500000005] [--from-state-id 500000045] [--done-state-id 500000010] ---dry-run prints exactly what would be transitioned and mutates nothing; it -does not require SHORTCUT_API_TOKEN. Without --dry-run, a missing -SHORTCUT_API_TOKEN is a hard error (exit 1) raised before any story is -touched. A per-story API failure is logged and never aborts the loop and -never fails the process as a whole — the deploy already happened, so this -script must never be the thing that turns a green rollout red. - -Prints a summary JSON at the end with counts and ids for each bucket: -transitioned, already_done, skipped_other_state, skipped_different_workflow, -failed — plus a skipped_detail list carrying each skipped story's id, -workflow_id, workflow_state_id and the reason it was skipped, and hydrated / -unresolved_story_ids echoed straight from the input file so stage 3 doesn't -silently lose stories that shipped but that shipped_stories.py could not -look up. - -Whenever the input's "stories" list is non-empty and at least one story was -skipped for a reason other than already being Done, this script prints a -WARNING to stderr naming those stories -- even in a MIXED release where some -other stories transitioned fine, so a partial skip is never invisible just -because the run "worked". If, in addition, NOTHING was transitioned at all, -that combination is a silent no-op -- the exact failure mode this script -exists to prevent -- and, unless --dry-run was passed, the process exits -non-zero so an unattended run can't look successful when it silently did -nothing. (--dry-run never exits non-zero for this: it is a preview, and -dry-run "transitioned" is inherently just a list of candidates, not evidence -anything actually happened or failed to.) - -This script only ever writes a story's workflow state -- it does not post -any comment or other annotation ("just mark it as done" is the whole job). - -All ids shown in this file's docstring and comments (e.g. story id 22222) -are placeholders, not real Shortcut story ids. The workflow and state ids -(500000005, 500000045, 500000010, 500000728) are real Shortcut workflow/ -state ids, not story ids, and are not covered by that placeholder rule. +Requires SHORTCUT_API_TOKEN unless --dry-run is passed. """ import argparse @@ -67,12 +19,7 @@ SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" -# Sefaria's "Standard" Shortcut workflow: "Deploy Ready" -> "Done". These are -# used as defaults so the script is runnable without extra flags in the -# common case; pass --workflow-id/--from-state-id/--done-state-id explicitly -# to override for a different workflow. The org has roughly ten workflows, -# each with its own state ids — e.g. at least one other workflow has its own -# Done state, 500000728, which is NOT the same as DEFAULT_DONE_STATE_ID. +# Sefaria's "Standard" Shortcut workflow: "Deploy Ready" -> "Done". DEFAULT_WORKFLOW_ID = 500000005 DEFAULT_FROM_STATE_ID = 500000045 DEFAULT_DONE_STATE_ID = 500000010 @@ -88,17 +35,7 @@ def warn(message: str) -> None: def classify_stories(stories, workflow_id, from_state_id, done_state_id): - """Split stories into (to_transition, already_done, skipped_other_state, - skipped_different_workflow) buckets. Pure function, no I/O. - - Workflow membership is checked BEFORE state, so a story sitting at a - different workflow's own Done state (e.g. 500000728) is classified as - skipped_different_workflow, not already_done — those state ids are not - interchangeable across workflows. A story with no workflow_id on record - (e.g. from a shipped-stories.json produced before this field existed) - falls through to the state-only checks instead of being flagged as a - mismatch, so older input files don't spuriously warn on every story. - """ + """Split stories into (to_transition, already_done, skipped_other_state, skipped_different_workflow) buckets.""" to_transition = [] already_done = [] skipped_other_state = [] @@ -159,8 +96,7 @@ def _ids(stories): def _skipped_detail(skipped_other_state, skipped_different_workflow): - """Build the human-readable detail list for every skipped (non-already-done) - story: id, workflow_id, workflow_state_id and why it was skipped.""" + """Build the detail list for every skipped (non-already-done) story: id, workflow_id, workflow_state_id, and reason.""" detail = [] for story in skipped_different_workflow: detail.append({ @@ -257,13 +193,6 @@ def main(): print(json.dumps(summary, indent=2)) - # Make silent skips impossible: any story skipped for a reason other than - # already being Done must be visible, whether or not anything else in - # this run transitioned. A MIXED release -- some stories transition - # fine, others get skipped for a different workflow/state -- used to - # produce no WARNING at all as long as `transitioned` was non-empty; that - # silently lost visibility into exactly the stories this script exists - # to flag. Only the "nothing happened at all" case is fatal. if stories and skipped_detail: skipped_ids_and_context = [ f"{d['id']} (workflow_id={d['workflow_id']}, workflow_state_id={d['workflow_state_id']}, " diff --git a/build/ci/merge_release_backfill.py b/build/ci/merge_release_backfill.py index bd26f40b9b..68a9f48092 100644 --- a/build/ci/merge_release_backfill.py +++ b/build/ci/merge_release_backfill.py @@ -1,65 +1,13 @@ #!/usr/bin/env python3 """ -Merge reconcile_deploy_ready.py's "shipped in THIS release" backfill stories -into shipped_stories.py's own output, for the release-notes prose step. - -Why this exists: reconcile_deploy_ready.py's org-wide sweep exists because -nothing else revisits a Deploy Ready story once it falls outside the -current release's git-range scan (RC2). Most of what it finds shipped in -EARLIER releases, and must never reach today's announcement -- that's why -its report is written to $RUNNER_TEMP and never touched by the prose step -(see that script's own docstring). But some of what it finds -- a race, a -discovery gap RC1 didn't close, a story that just never got the write-back -it deserved -- shipped in the CURRENT release, same as everything -shipped_stories.py's own git-range scan already found. Silently excluding -THOSE from the announcement is a different flavor of the same underlying -mistake ("say what actually shipped today"), just by omission instead of -leakage. This script is the one, single, deterministic place that decides -which backfilled stories cross that line -- never the prose agent itself, -never a prompt instruction. - -The decision was ALREADY MADE upstream, in reconcile_deploy_ready.py's own -classify_candidates: a shipped entry there carries `hydrated_story` if and -only if its `shipping_release_tag` (the TRUE release, resolved via -resolve_shipping_release_tag's `git tag --contains` lookup) equals the -CURRENT prod tag reconcile_deploy_ready.py was run against. This script -trusts that signal completely and does nothing else -- it is a pure, -mechanical merge, not a second opinion. `hydrated_story` is already in -shipped_stories.py's own hydrated-story shape (id, name, description, url, -workflow_id, workflow_state_id, story_type), built by -reconcile_deploy_ready.py from data it already had in memory -- no extra -Shortcut API call here either. - -Reads --shipped-stories-out (shipped_stories.py's own --out file) and ---reconcile-report (reconcile_deploy_ready.py's own --out file), and writes -the merged result to --out (default: overwrite --shipped-stories-out in -place, so the prose step's existing "read shipped-stories.json" step needs -no change at all). Deduplicates by story id against what shipped_stories.py -ALREADY found -- a story RC1's text/PR-link discovery independently found -in this same range must not be double-counted or duplicated in the merged -`stories` list. - -CRITICAL: this script only ever reads reconcile_deploy_ready.py's report to -extract the pre-filtered, pre-decided `hydrated_story` entries. It never -reads (or writes back) `shipped`/`pending`/`triage` wholesale, and it never -makes its own judgment about which release a story belongs to -- that -judgment already happened, once, in reconcile_deploy_ready.py, using git -evidence. Running this script does not, by itself, change what's readable -by the release-notes prose step: it only ever touches shipped-stories.json, -which was already that step's designated input before this script existed. -The reconcile report itself must still never be passed to, or made -readable by, that step -- this script's whole job is to extract the one -safe, pre-filtered slice from it and leave the rest behind in $RUNNER_TEMP. +Merges reconcile_deploy_ready.py's "shipped in this release" backfill +stories into shipped_stories.py's own output, for the release-notes prose +step. Only reads the pre-filtered `hydrated_story` entries from the +reconcile report; never touches `shipped`/`pending`/`triage` wholesale. Usage: python3 merge_release_backfill.py --shipped-stories-out shipped-stories.json \ --reconcile-report reconcile-deploy-ready-report.json [--out shipped-stories.json] - -Stdlib only -- no third-party dependencies, matching the rest of this -pipeline's scripts. - -All story ids in this file's docstring and comments (e.g. 11111) are -placeholders, not real Shortcut story ids. """ import argparse @@ -77,28 +25,12 @@ def warn(message: str) -> None: def backfill_stories_from_report(report): - """Every `hydrated_story` already attached to report['shipped'] -- - i.e. every story reconcile_deploy_ready.py ALREADY determined belongs - in the CURRENT release (shipping_release_tag == the prod tag that run - was against). This function makes no decision of its own: a shipped - entry with no `hydrated_story` (an earlier or unresolvable release) is - silently skipped, exactly as it should be -- fail closed, never guess - a story into an announcement this script didn't independently verify - and has no way to.""" + """Every `hydrated_story` already attached to report['shipped'] -- entries with none are skipped.""" return [s["hydrated_story"] for s in (report.get("shipped") or []) if s.get("hydrated_story")] def merge(shipped_stories_data, backfill_stories): - """Fold `backfill_stories` (already in shipped_stories.py's own - hydrated-story shape) into `shipped_stories_data`'s `story_ids` / - `stories`, deduplicated by story id. A story shipped_stories.py's own - git-range + RC1 discovery ALREADY found independently must not be - duplicated -- existing entries always win the dedupe (this script - never overwrites data shipped_stories.py already hydrated for itself). - - Returns (merged_data, added_ids) so the caller can report exactly what - was added, for visibility -- mirrors shipped_stories.py's own - stories_from_shortcut_pr_link provenance field.""" + """Fold `backfill_stories` into `shipped_stories_data`'s `story_ids` / `stories`, deduplicated by story id. Returns (merged_data, added_ids).""" existing_ids = set(str(sid) for sid in (shipped_stories_data.get("story_ids") or [])) stories = list(shipped_stories_data.get("stories") or []) added_ids = [] @@ -113,10 +45,6 @@ def merge(shipped_stories_data, backfill_stories): shipped_stories_data["story_ids"] = sorted(existing_ids, key=int) shipped_stories_data["stories"] = stories - # Provenance: which ids were folded in by the reconciliation sweep, - # distinct from shipped_stories.py's own git-range/RC1 discovery -- - # same purpose as that script's own stories_from_shortcut_pr_link - # field (say what only the sweep knew). shipped_stories_data["stories_from_reconciliation_backfill"] = sorted(added_ids, key=int) return shipped_stories_data, added_ids diff --git a/build/ci/reconcile_deploy_ready.py b/build/ci/reconcile_deploy_ready.py index 622ef04ad4..f157ceff5b 100644 --- a/build/ci/reconcile_deploy_ready.py +++ b/build/ci/reconcile_deploy_ready.py @@ -1,152 +1,20 @@ #!/usr/bin/env python3 """ -Reconcile "Deploy Ready" Shortcut stories against what has actually reached -prod, regardless of which release shipped them (RC2 in the incident -writeup). - -`shipped_stories.py` + `mark_stories_deployed.py` only ever look at the -commit range for ONE release (prev-tag..cur-tag). A story whose PR merged -and shipped in an EARLIER release -- or before this pipeline existed at all --- never gets revisited: nothing ever walks backward and asks "is this -Deploy Ready story actually done?". Of 20 stories stuck in Deploy Ready when -this was diagnosed, 10 were this class of bug versus 1 for the git-only -discovery gap (RC1, fixed in shipped_stories.py) -- this is the dominant -defect. - -This script is a standalone, org-wide sweep: it enumerates EVERY -non-archived Deploy Ready story (not just stories that happen to fall in -some git range), resolves each one's linked, merged PR(s), and checks -whether any of those PRs' merge commits actually landed in the current prod -tag. It classifies every story into exactly one of three buckets: - - - shipped -- at least one qualifying PR is an ancestor of the prod tag. - Transitioned Deploy Ready -> Done (500000045 -> 500000010). - - pending -- has a qualifying merged PR, but none of them are in prod - yet. Left alone -- this is the CORRECT state, not a bug. - - triage -- no qualifying PR at all (nothing merged, wrong repo, a - promotion PR instead of the real one, ...), or the story - lives in a non-Standard Shortcut workflow where the Deploy - Ready/Done state ids used here don't even apply. Left - alone and reported for a human to look at -- this is the - part of the output that actually needs eyes on it. - -Five guards apply before a linked PR counts as evidence a story shipped -- -each one caught a real false positive while this script was being built (or -maintained) against live data: - - 1. repository_id must be Sefaria-Project's (500000103). A story can link - a PR from a DIFFERENT repo (e.g. a docs or infra repo); resolving that - PR number against Sefaria-Project instead finds an unrelated, often - much older, PR that happens to share the number -- and that PR can - easily already be in prod. Skipping this guard silently reports "in - prod" for a story that never shipped anything to Sefaria-Project at - all. - 2. target_branch_name must be "master". Some stories link a PROMOTION PR - (preprod -> prod, or master -> preprod) instead of, or alongside, the - actual feature PR. A promotion PR merges constantly and proves nothing - about whether THIS story's own change reached prod. - 3. The PR's own HEAD (source) branch must NOT be a long-lived environment - branch (master, preprod, prod). Verified live: a promotion PR merging - preprod INTO master passes guards 1-2 and 4 cleanly -- it's merged, - against the right repo, and its target really is "master" -- yet it's - still a promotion merge, not a feature PR. Guard 2 alone cannot catch - this shape (a promotion merge legitimately targets master); only the - HEAD branch gives it away. This is the exact live false positive that - was found and fixed: a story was classified `shipped` on the strength - of a promotion PR shaped exactly this way, while its genuine feature - PR (correctly) failed guard 2 for targeting a hotfix branch instead of - master directly. - 4. merged must be true. An open or closed-without-merging PR is not - evidence of anything having shipped. - 5. workflow_id must be Sefaria's "Standard" workflow (500000005), and - workflow_state_id must be exactly the numeric Deploy Ready state id - (500000045) within it. The Shortcut state named "Deploy Ready" (note: - the real name carries a trailing space, "Deploy Ready ") is - workflow-specific -- 500000045 does not exist as a concept in, say, - the Content workflow (500000061). A story living in a different - workflow that the search endpoint still matched by state NAME must - never be transitioned using Standard's state ids; it's routed to - triage with its actual workflow/state ids reported instead, mirroring - mark_stories_deployed.py's skipped_different_workflow handling. - -Guards 1-4 are PR-level and live in shortcut_pr_guards.py, shared with -shipped_stories.py's RC1 fallback -- see that module's own docstring. Guard -5 is story-level and specific to this sweep; it stays local, in -classify_stories below. - -Enumeration uses the token'd search endpoint -(`search/stories?query=state:"Deploy Ready" !is:archived`), paginated via -its `next` cursor -- NOT `iterations-get-active`, which is silently scoped -to the calling token's own teams and has already produced an incomplete -picture for this team once. The search endpoint returned all 20 stuck -stories across 4 different teams in testing; iterations-get-active would -have missed most of them. - -For each qualifying PR, its merge commit SHA is resolved via -`gh pr view --json mergeCommit` and tested with -`git merge-base --is-ancestor ` against the newest `prod/*` -tag (by `--sort=-creatordate`) unless `--prod-tag` overrides it. This is the -same ancestry check verified against live data to correctly discriminate -merged-but-not-yet-promoted PRs (not an ancestor) from ones that already -reached prod (an ancestor) -- `git log --grep="(#N)"` was tried and -rejected: it misses squash-merge subjects and can't tell a real promotion -merge apart from an unrelated one. - ---dry-run is the default (matching mark_stories_deployed.py's posture, only -stricter): this is a bulk mutation of shared team state across potentially -many stories and several different teams, and unlike a single release's -handful of stories, there's no natural moment (a deploy) that makes running -it low-risk. Nothing is ever transitioned without an explicit --apply. - -Emits a JSON report (--out) with all three buckets in full, plus a readable -stdout summary. CRITICAL: this script never reads or writes -shipped-stories.json itself and never talks to the release-notes prose -step directly -- the stories it backfills mostly shipped in EARLIER -releases, and leaking them into today's release announcement would have -Slack claim a dozen old features shipped today. This script only ever -transitions Shortcut state; it never posts a comment or any other -annotation ("just mark it as done" is the whole job here). - -One exception exists, and it is handled by a SEPARATE, later, explicitly -filtered step -- never by this script writing to shipped-stories.json -itself: a story this sweep backfills can, in the ordinary case, ALSO have -shipped in the CURRENT release rather than an earlier one (the sweep exists -because nothing else revisits Deploy Ready stories, including ones from -THIS release that a race or a discovery gap missed). That case belongs in -today's announcement -- silently excluding it would just be a different -flavor of the same failure this whole separation defends against, this -time by omission instead of leakage. So every shipped entry in this -report also carries `shipping_release_tag`: the TRUE release that shipped -it, resolved via `resolve_shipping_release_tag()` (`git tag --list -'prod/*' --contains --sort=creatordate | head -1` -- the -first, earliest-created, prod/* tag that actually contains the winning -PR's merge commit). When `shipping_release_tag` equals the CURRENT -`prod_tag`, the entry ALSO carries `hydrated_story` -- a story object in -shipped_stories.py's own hydrated-story shape, built from data this sweep -already fetched (no extra API call) -- so a separate, deterministic -downstream script (`merge_release_backfill.py`) can fold exactly that -story, and only that story, into shipped-stories.json before the prose -step runs. Every other shipped entry, and the reconcile report as a -whole, is never read by that step or any step after it. `resolve_ -shipping_release_tag()` returns None when it can't determine a release at -all (shallow checkout, tag history gap, ...); that case is EXCLUDED from -`hydrated_story` too -- fail closed, never guess a story into an -announcement. +Reconciles "Deploy Ready" Shortcut stories against what has actually +reached prod, org-wide and regardless of which release shipped them. + +Enumerates every non-archived Deploy Ready story, resolves each one's +linked, merged PR(s), and checks whether any of those PRs' merge commits +landed in the current prod tag. Classifies each story as shipped +(transitioned Deploy Ready -> Done), pending (qualifying PR not yet in +prod), or triage (no qualifying PR, or a non-Standard workflow/state). Usage: python3 reconcile_deploy_ready.py [--dry-run] python3 reconcile_deploy_ready.py --apply python3 reconcile_deploy_ready.py --apply --prod-tag prod/6.111.0-prod.2+chart.0.87.5-prod.1 --out reconcile-report.json -Requires `git` and `gh` on PATH (same as shipped_stories.py) and -SHORTCUT_API_TOKEN -- required even for --dry-run, since enumeration and -the PR<->story evidence both come from the Shortcut API, not from git. - -All story ids shown in this file's docstring and comments (e.g. 11111, -22222) are placeholders, not real Shortcut story ids. The repo/workflow/ -state ids (500000103, 500000005, 500000045, 500000010, 500000061) are real -Shortcut/GitHub ids, not story ids, and are not covered by that placeholder -rule -- same convention as shipped_stories.py and mark_stories_deployed.py. +Requires `git` and `gh` on PATH, and SHORTCUT_API_TOKEN (required even for --dry-run). """ import argparse @@ -159,41 +27,22 @@ import urllib.parse import urllib.request -# The PR-level shipping-evidence guards (merged / right repo / right target -# branch) are shared with shipped_stories.py's RC1 PR-link fallback -- see -# shortcut_pr_guards.py's own docstring for why a single shared -# implementation matters here (a promotion PR is exactly as good at fooling -# either script, and the two must never silently drift apart on what counts -# as evidence). build/ci is not a package (see tests/conftest.py), but a -# plain sibling-module import works both when this file is run directly -# (python3 puts its own directory on sys.path[0]) and under pytest (the test -# conftest adds build/ci to sys.path the same way). +# build/ci is not a package (see tests/conftest.py); a plain sibling-module +# import works both run directly and under pytest. import shortcut_pr_guards SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" SHORTCUT_API_ROOT = "https://api.app.shortcut.com" -# Sefaria's "Standard" Shortcut workflow -- same ids mark_stories_deployed.py -# defaults to. Kept as separate constants here (not imported) because these -# CI scripts are deliberately standalone, single-file tools -- see the -# existing die()/warn() duplication between shipped_stories.py and -# mark_stories_deployed.py, which follows the same convention. (The PR-level -# guard constants/functions are the one deliberate exception -- see the -# shortcut_pr_guards import above.) +# Sefaria's "Standard" Shortcut workflow. STANDARD_WORKFLOW_ID = 500000005 DEPLOY_READY_STATE_ID = 500000045 DONE_STATE_ID = 500000010 -# Guard #1/#2 defaults -- re-exported from shortcut_pr_guards so the rest of -# this file (and its tests) can keep referring to them by their existing -# names here. SEFARIA_PROJECT_REPO_ID = shortcut_pr_guards.SEFARIA_PROJECT_REPO_ID DEFAULT_TARGET_BRANCH = shortcut_pr_guards.DEFAULT_TARGET_BRANCH DEFAULT_REPO = "Sefaria/Sefaria-Project" -# `branch:"..."` and `pull-request:N` do NOT resolve a PR to its story on -# this Shortcut org -- verified empirically. `pr:N` is the only search -# operator that works, both here and in shipped_stories.fetch_story_by_pr_link. DEPLOY_READY_SEARCH_QUERY = 'state:"Deploy Ready" !is:archived' @@ -207,11 +56,7 @@ def warn(message: str) -> None: def run_git(args): - """Run a git subcommand, returning stdout. Exits the process on failure. - Mirrors shipped_stories.run_git -- used here only for the plain, - always-expected-to-succeed prod-tag listing; the ancestry check below - needs its own non-fatal handling of git's exit codes, so it does not go - through this helper.""" + """Run a git subcommand, returning stdout. Exits the process on failure.""" proc = subprocess.run(["git", *args], capture_output=True, text=True) if proc.returncode != 0: die(f"git {' '.join(args)} failed: {proc.stderr.strip()}") @@ -219,10 +64,7 @@ def run_git(args): def resolve_default_prod_tag(): - """Newest prod/* tag by creation date, the same ordering - shipped_stories.py uses for --version resolution. Dies if there are no - prod/* tags at all -- with none, there's no prod state to reconcile - against, and --prod-tag can be passed explicitly if that's ever wrong.""" + """Newest prod/* tag by creation date. Dies if there are no prod/* tags at all.""" tags = [t for t in run_git(["tag", "--list", "prod/*", "--sort=-creatordate"]).splitlines() if t.strip()] if not tags: die("No 'prod/*' tags found in this checkout; pass --prod-tag explicitly.") @@ -230,12 +72,7 @@ def resolve_default_prod_tag(): def search_deploy_ready_stories(token): - """Enumerate ALL non-archived Deploy Ready stories via the token'd - search endpoint, paginated via its `next` cursor. Deliberately NOT - `iterations-get-active`: that endpoint is silently scoped to the - calling token's own teams, while `search/stories` returned all 20 - stuck stories across 4 different teams when this was verified live -- - org-wide is exactly what "every Deploy Ready story" requires.""" + """Enumerate all non-archived Deploy Ready stories via the token'd search endpoint, paginated via its `next` cursor. Deliberately not `iterations-get-active`, which is scoped to the calling token's own teams.""" stories = [] next_path = f"/api/v3/search/stories?query={urllib.parse.quote(DEPLOY_READY_SEARCH_QUERY)}" while next_path: @@ -248,21 +85,12 @@ def search_deploy_ready_stories(token): return stories -# gather_linked_prs / qualifying_prs (guards #1-3) now live in -# shortcut_pr_guards.py, shared with shipped_stories.py's RC1 fallback -- -# re-exported here under their existing names so nothing else in this file -# (or its tests) needs to change. Guard #4 (workflow/state) is a -# story-level check specific to the Deploy Ready sweep and stays local to -# classify_stories below. gather_linked_prs = shortcut_pr_guards.gather_linked_prs qualifying_prs = shortcut_pr_guards.qualifying_prs def fetch_pr_merge_oid(pr_number, repo): - """Resolve a merged PR's merge commit SHA via `gh pr view --json - mergeCommit`. Mirrors shipped_stories.fetch_pr_branch's error posture: a - failed lookup is logged and returns None rather than raising -- one - story's PR being unreachable must never abort the whole sweep.""" + """Resolve a merged PR's merge commit SHA via `gh pr view --json mergeCommit`. A failed lookup is logged and returns None.""" proc = subprocess.run( ["gh", "pr", "view", str(pr_number), "--repo", repo, "--json", "mergeCommit"], capture_output=True, @@ -281,8 +109,7 @@ def fetch_pr_merge_oid(pr_number, repo): def fetch_merge_oids(pr_numbers, repo, max_workers=8): - """Batch-resolve fetch_pr_merge_oid across every qualifying PR in this - sweep concurrently -- same pattern as shipped_stories.fetch_branches.""" + """Batch-resolve fetch_pr_merge_oid across every qualifying PR concurrently.""" oid_by_pr = {} if not pr_numbers: return oid_by_pr @@ -292,8 +119,6 @@ def fetch_merge_oids(pr_numbers, repo, max_workers=8): try: pr_number, oid = future.result() except FileNotFoundError: - # Every other in-flight lookup will hit the same error -- - # fail fast with one clear message instead of N tracebacks. die("`gh` was not found on PATH. Install the GitHub CLI " "(https://cli.github.com/) or ensure it's available in " "this environment; PR merge-commit lookups cannot proceed without it.") @@ -303,12 +128,7 @@ def fetch_merge_oids(pr_numbers, repo, max_workers=8): def is_ancestor_of_prod(oid, prod_tag): - """True if commit `oid` reached prod (is an ancestor of `prod_tag`), - False if git can definitively say it did not. Returns None if the check - itself couldn't run (e.g. a shallow checkout that never fetched `oid`) - -- that's a local-repo problem, not proof either way, and must not be - conflated with a confirmed "not in prod" (which would misclassify a - story that may well have shipped as merely pending).""" + """True if commit `oid` is an ancestor of `prod_tag`, False if git can definitively say it is not, None if the check itself couldn't run (a local-repo problem, not proof either way).""" proc = subprocess.run( ["git", "merge-base", "--is-ancestor", oid, prod_tag], capture_output=True, @@ -323,21 +143,12 @@ def is_ancestor_of_prod(oid, prod_tag): def resolve_shipping_release_tag(oid): - """The TRUE release that shipped commit `oid`: the first (earliest - created) `prod/*` tag that actually contains it. Used to decide - whether a backfilled story belongs in TODAY's release notes -- see the - module docstring's `shipping_release_tag` / `hydrated_story` - paragraph and `classify_candidates` below. - - `--sort=creatordate` (ascending, oldest first) is deliberate and the - OPPOSITE of resolve_default_prod_tag's `-creatordate` above -- that one - wants the newest tag (today's release); this one wants the OLDEST tag - that still contains the commit, i.e. the first release it ever - reached. Returns None if the lookup fails outright (non-fatal -- - logged, and the caller must fail closed: never guess or default to - "belongs to the current release" when this is unresolvable) or if no - prod/* tag contains it at all (e.g. a shallow checkout, or a genuine - gap in tag history).""" + """The first (earliest created) `prod/*` tag that contains commit `oid` -- the release that actually shipped it. + + `--sort=creatordate` (ascending) is deliberate and the opposite of + resolve_default_prod_tag's `-creatordate` above: that one wants the + newest tag, this one wants the oldest tag that still contains the + commit. Returns None if the lookup fails or no prod/* tag contains it.""" proc = subprocess.run( ["git", "tag", "--list", "prod/*", "--contains", oid, "--sort=creatordate"], capture_output=True, @@ -351,10 +162,7 @@ def resolve_shipping_release_tag(oid): def transition_story(story_id, done_state_id, token): - """PUT a workflow_state_id update to Shortcut. Mirrors - mark_stories_deployed.transition_story exactly (kept as its own copy - here rather than imported, for the same standalone-script reason the - die()/warn() duplication above follows).""" + """PUT a workflow_state_id update to Shortcut.""" url = f"{SHORTCUT_API_BASE}/stories/{story_id}" body = json.dumps({"workflow_state_id": done_state_id}).encode("utf-8") req = urllib.request.Request(url, data=body, method="PUT") @@ -392,13 +200,7 @@ def _story_summary(story): def _triage_context(story): - """Extra context for a TRIAGE story only, already sitting in the same - Shortcut search response that produced `story` -- adds no extra API - calls ("cheap", as opposed to e.g. an extra `gh pr view` round trip - per triage story, which this deliberately avoids). This is the raw - material a human (or the opt-in triage-explainer workflow step - downstream -- see build/ci/triage_explainer.py) needs to propose why a - story is stuck, without re-deriving it from scratch.""" + """Extra context for a triage story only, already present in the search response (no extra API call).""" return { "description": story.get("description"), "comments": [c.get("text") for c in (story.get("comments") or []) if c.get("text")], @@ -406,11 +208,7 @@ def _triage_context(story): def _diagnose_linked_pr(pr, repo_id, target_branch): - """Which of the four PR-level guards (see qualifying_prs / - shortcut_pr_guards.passes_pr_guards) this specific linked PR fails, if - any. Diagnostic ONLY -- classification itself never reads this; it - exists purely so a triage story's report entry can say WHY a linked PR - didn't count instead of just listing its bare number.""" + """Which PR-level guards this linked PR fails, if any. Diagnostic only -- classification never reads this.""" failed = [] if pr.get("merged") is not True: failed.append("not merged") @@ -424,19 +222,7 @@ def _diagnose_linked_pr(pr, repo_id, target_branch): def classify_stories(stories, repo_id, target_branch): - """Pure classification, no I/O beyond what's already embedded in the - Shortcut story payloads: split into (triage, candidates), where - candidates is a list of (story, qualifying_prs) pairs still needing a - prod-ancestry check. Guard #4 (workflow + numeric state id) is applied - first and unconditionally routes to triage -- a story on any workflow - other than Standard, or sitting at any state id other than the numeric - Deploy Ready id (500000045) despite matching the "Deploy Ready" state - NAME search, must never reach the PR guards or a transition at all. - - Every triage entry also carries _triage_context (description, comment - text) -- shipped/pending entries deliberately do NOT, so they stay as - lean as before this was added; only a triage story's own report entry - ever needs to answer "why is this one stuck".""" + """Split stories into (triage, candidates), where candidates is a list of (story, qualifying_prs) pairs still needing a prod-ancestry check. A story not on the Standard workflow/state routes straight to triage.""" triage = [] candidates = [] for story in stories: @@ -463,10 +249,6 @@ def classify_stories(stories, repo_id, target_branch): entry["linked_pr_numbers"] = sorted( pr.get("number") for pr in linked if pr.get("number") is not None ) - # Full per-PR diagnostic, sorted the same way as - # linked_pr_numbers above for a stable, readable report -- - # linked_pr_numbers stays as-is (existing consumers rely on - # it); this is additive. entry["linked_prs"] = [ {"number": pr.get("number"), "failed_guards": _diagnose_linked_pr(pr, repo_id, target_branch)} for pr in sorted(linked, key=lambda p: (p.get("number") is None, p.get("number"))) @@ -480,14 +262,7 @@ def classify_stories(stories, repo_id, target_branch): def _story_for_release_notes(story): - """Build a story entry in EXACTLY shipped_stories.py's own - hydrated-story shape (id, name, description, url, workflow_id, - workflow_state_id, story_type -- see that script's fetch_story) - directly from the story object this sweep already fetched via - search/stories -- its "detail=full" response already carries every one - of these fields, so no extra Shortcut API call is needed here. Used - ONLY for a shipped story whose `shipping_release_tag` equals the - CURRENT prod tag -- see classify_candidates and the module docstring.""" + """Build a story entry in shipped_stories.py's own hydrated-story shape, from data already fetched (no extra API call).""" return { "id": story.get("id"), "name": story.get("name"), @@ -500,27 +275,12 @@ def _story_for_release_notes(story): def classify_candidates(candidates, oid_by_pr, prod_tag): - """For each (story, qualifying_prs) candidate, check every qualifying - PR's merge commit for prod ancestry and split into shipped / pending. - A PR whose merge oid never resolved, or whose ancestry check itself - couldn't run, counts as inconclusive -- never as "in prod" -- so a - resolution failure can only ever push a story toward pending (leave it - alone), never wrongly toward shipped (a mutation). - - Every shipped entry also carries `shipping_release_tag` -- the TRUE - release that shipped it (resolve_shipping_release_tag, from the first - confirmed-in-prod qualifying PR's merge commit) -- computed - unconditionally, independent of --apply/--dry-run: this is a fact - about git history, not about whether THIS run happens to mutate - Shortcut, so a dry-run report is exactly as trustworthy an input to - the downstream release-notes merge decision as a live one. ONLY when - `shipping_release_tag` equals the CURRENT `prod_tag` -- i.e. this - story provably shipped in TODAY's release, not an earlier one -- the - entry ALSO carries `hydrated_story` (see _story_for_release_notes). - That field's presence is the ONLY signal merge_release_backfill.py - uses to decide whether to fold a backfilled story into - shipped-stories.json; every other shipped entry (a different or - unresolvable release) carries no such field and is never merged.""" + """For each (story, qualifying_prs) candidate, check every qualifying PR's merge commit for prod ancestry and split into shipped / pending. + A PR whose merge oid never resolved, or whose ancestry check couldn't run, counts as inconclusive, never as "in prod". + + Every shipped entry carries `shipping_release_tag`, the true release that shipped it, computed unconditionally regardless of + --apply/--dry-run. Only when it equals the current `prod_tag` does the entry also carry `hydrated_story` -- the only signal + merge_release_backfill.py uses to fold a backfilled story into shipped-stories.json.""" shipped = [] pending = [] for story, prs in candidates: @@ -548,9 +308,7 @@ def classify_candidates(candidates, oid_by_pr, prod_tag): def print_summary(report): - """Readable stdout summary. The triage list is printed in FULL, not - just counted -- it's the part of this report a human actually has to - act on.""" + """Readable stdout summary. The triage list is printed in full, not just counted.""" counts = report["counts"] print(f"Prod tag: {report['prod_tag']}") print(f"Mode: {'APPLY (mutating)' if report['applied'] else 'dry-run (no mutation)'}") @@ -568,10 +326,6 @@ def print_summary(report): status = "transitioned" else: status = f"FAILED: {s.get('transition_error')}" - # belongs-to-current-release marker: hydrated_story's presence is - # the same signal merge_release_backfill.py acts on downstream -- - # printed here too so a human reading this summary can see which - # shipped stories will (or won't) show up in today's announcement. release_note = ( "belongs in THIS release's notes" if s.get("hydrated_story") else f"shipped in {s.get('shipping_release_tag')}" if s.get("shipping_release_tag") diff --git a/build/ci/shipped_stories.py b/build/ci/shipped_stories.py index 6458825ec6..a0344424eb 100755 --- a/build/ci/shipped_stories.py +++ b/build/ci/shipped_stories.py @@ -1,63 +1,22 @@ #!/usr/bin/env python3 """ -Deterministically resolve which Shortcut stories shipped in a prod rollout. - -Walks the git tree between two prod tags (or an explicit commit range), -extracts Shortcut (SC) story ids from commit subjects and, for commits that -reference a merged PR, from that PR's branch name too. As a THIRD discovery -source, a commit whose PR carries no sc-NNNNN id in either place (subject or -branch name) is looked up against Shortcut's own PR<->story link -(`search/stories?query=pr:`) when SHORTCUT_API_TOKEN is set -- git text is -not the only place a story/PR link can live; a story can be linked to a PR -from the Shortcut UI without the PR's branch ever mentioning a story code. -This fallback is NOT run at all for a commit whose subject is auto-generated -merge/branch-sync noise (NOISE_PATTERN) or whose PR's own head branch is a -long-lived environment branch (master/preprod/prod) -- both are shapes a -promotion PR takes, and a promotion PR resolves via `pr:` to a real story -just as readily as that story's actual feature PR does, while proving -nothing about whether that story's own change shipped (verified live: PRs -whose head branch was `preprod` or `master` each resolved to a real story -this way). The single search result is also re-checked against the SAME -four PR-level guards `reconcile_deploy_ready.py`'s org-wide sweep applies -(merged / Sefaria-Project repo / target branch master / head branch not a -long-lived environment branch; shared via shortcut_pr_guards.py so the two -scripts cannot silently drift apart) before its id is adopted -- a match -that fails those guards is a warn-and-skip, not a fallback of last resort. -The id is adopted ONLY when the search resolves to EXACTLY one story AND -that story's PR passes those guards; ids recovered this way are also -surfaced separately in `stories_from_shortcut_pr_link` so a report can say -what only Shortcut knew. -Revert commits (`Revert "..."`, `Revert: ...`, `revert(...)`) never -contribute story ids to the shipped set; their suppressed ids are surfaced -separately in `reverted_commits` instead of being silently dropped. -Optionally hydrates each id via the Shortcut API (id, name, description, -url, workflow id, workflow state, story type) when SHORTCUT_API_TOKEN is -set — `workflow_id` is included because a workflow's Done state id is not -universal across Shortcut workflows, and downstream tooling -(mark_stories_deployed.py) needs it to tell "different workflow" apart from -"different state". Emits a single JSON document that downstream tooling -(the sefaria-release-notes skill, mark_stories_deployed.py) consumes — this -script never writes prose and never mutates a Shortcut story. +Deterministically resolves which Shortcut stories shipped in a prod rollout, +by walking the git range between two prod tags and extracting story ids +from commit subjects, PR branch names, and (as a fallback) Shortcut's own +PR<->story link. Usage: python3 shipped_stories.py --version 6.111.0-prod.2 [--out shipped-stories.json] [--repo Sefaria/Sefaria-Project] python3 shipped_stories.py --range .. [--out shipped-stories.json] [--repo Sefaria/Sefaria-Project] With --version V, the current tag is resolved as the single tag matching the -glob `prod/V+*` (an optional leading "v" on V is stripped first), and the -previous tag is whichever prod/* tag immediately precedes it by creation -date. With --range, the two endpoints are used verbatim as given. - -Requires `git` and `gh` on PATH. `gh` is only used to look up PR branch -names (`gh pr view --json headRefName,number`) and is never required to -succeed — a failing lookup for one PR is logged to stderr and skipped. -SHORTCUT_API_TOKEN is optional; without it, story ids are still emitted but -`stories` is empty and `unresolved_story_ids` is not populated (hydration -was never attempted, which is a different case from a failed lookup), and -the RC1 PR-link fallback above is skipped entirely (git-text discovery only). - -All ids shown in this file's docstring and comments (e.g. story id 11111) -are placeholders, not real Shortcut story ids. +glob `prod/V+*`, and the previous tag is whichever prod/* tag immediately +precedes it by creation date. With --range, the two endpoints are used +verbatim as given. + +Requires `git` and `gh` on PATH. SHORTCUT_API_TOKEN is optional; without +it, story ids are still emitted but hydration and the PR-link fallback are +skipped. """ import argparse @@ -71,46 +30,15 @@ import urllib.parse import urllib.request -# The PR-level shipping-evidence guards (merged / right repo / right target -# branch) are shared with reconcile_deploy_ready.py's org-wide sweep -- see -# shortcut_pr_guards.py's own docstring for why a single shared -# implementation matters (a promotion PR is exactly as good at fooling -# either script). build/ci is not a package (see tests/conftest.py), but a -# plain sibling-module import works both when this file is run directly -# (python3 puts its own directory on sys.path[0]) and under pytest (the -# test conftest adds build/ci to sys.path the same way). +# build/ci is not a package (see tests/conftest.py); a plain sibling-module +# import works both run directly and under pytest. import shortcut_pr_guards -# Long-lived environment branches. A PR whose HEAD branch is one of these is -# a promotion/branch-sync PR (preprod -> prod, master -> preprod, ...), not -# a real feature PR -- verified live to resolve via the RC1 PR-link fallback -# below to a real story despite proving nothing about whether that story's -# own change shipped. Checked against the PR's *head* branch (the same -# `branch` value already resolved via fetch_pr_branch for story-id -# extraction), not its target branch -- shortcut_pr_guards' target-branch -# guard covers that side separately. -# -# Re-exported from shortcut_pr_guards (guard #4 there) rather than a second -# local copy -- these two scripts already drifted apart once on whether a -# head-branch guard existed at all (reconcile_deploy_ready.py had none until -# a promotion PR slipped a story into "shipped" live); naming the same -# frozenset object in both places is what actually prevents a second drift, -# not just matching the values by hand. LONG_LIVED_ENV_BRANCHES = shortcut_pr_guards.LONG_LIVED_ENV_BRANCHES -# Shortcut (SC) story id patterns recognized in a commit subject or a PR -# branch name. Kept intentionally short: `\bsc[-_](\d+)\b` (pattern 1) has a -# TRAILING word boundary, so it already matches "sc-N"/"sc_N" wrapped in any -# punctuation (brackets, parens, a leading "chore:"/"feat:" etc.) -- adding a -# separate bracket/paren/prefix-scoped pattern for each of those shapes would -# just re-derive what pattern 1 already covers. The other two patterns here -# are kept because they are NOT subsumed by pattern 1: -# - `chore[:/\(].*?sc[-_](\d+)` / `feat[:/\(].*?sc[-_](\d+)` have no -# trailing boundary, so they still match when a word character follows -# the digits directly with no separator (e.g. a "chore(sc_123abc)"-shaped -# subject). -# - `feature/sc[-_ ](\d+)` allows a literal space after "sc", which -# pattern 1's `[-_]` does not. +# Pattern 1 has a trailing word boundary and covers "sc-N"/"sc_N" in most +# contexts; patterns 2-4 catch shapes it doesn't (no trailing boundary, or a +# literal space after "sc"). SC_PATTERNS = [ re.compile(r'\bsc[-_](\d+)\b', re.IGNORECASE), re.compile(r'feature/sc[-_ ](\d+)', re.IGNORECASE), @@ -120,38 +48,22 @@ PR_PATTERN = re.compile(r'\(#(\d+)\)') -# A real (non-squash) merge commit's subject never gets the parenthesized -# "(#N)" form -- GitHub writes it as a bare "Merge pull request #N from -# /". For a merge-commit PR this subject is often the ONLY -# place the PR number appears in the whole range (no matching child commit -# carries it), so PR_PATTERN alone would silently miss it. +# A real (non-squash) merge commit's subject uses the bare "Merge pull +# request #N from /" form instead of "(#N)". MERGE_PR_PATTERN = re.compile(r'^Merge pull request #(\d+)\s+from\s+\S+', re.IGNORECASE) -# Matches the double-quoted original subject inside a `Revert "..."` commit -# (e.g. a subject like `Revert "fix(sc-13): correct the thing"` captures -# `fix(sc-13): correct the thing`). Used to find that original commit -# elsewhere in the same range (see the reverted-original handling in -# main()) -- not applicable to the `Revert: ...` / `revert(...)` forms, -# which never quote a subject. +# Captures the quoted original subject inside a `Revert "..."` commit; not +# applicable to the `Revert: ...` / `revert(...)` forms. REVERT_QUOTE_PATTERN = re.compile(r'^revert\s+"(.+?)"', re.IGNORECASE) -# Auto-generated noise that should never count as "a real commit missing a -# story id" in commits_without_story: deploy() markers, "Merge pull -# request" / "Merge branch" / "Merge remote-tracking branch" subjects (the -# latter two are plain branch-sync merges -- now that merge commits are -# walked at all for ID/PR extraction, these show up in the range too and -# must not pollute that list), and any subject ending in a "[skip ci]" -# marker. Generalized from an enumerated (staging|preprod|prod) alternation -# so new deploy environments (e.g. deploy(sandbox)) are recognized without -# an edit here. +# Auto-generated noise excluded from commits_without_story. NOISE_PATTERN = re.compile( r'^(deploy\(\w+\)|Merge (pull request|branch|remote-tracking branch))|\[skip ci\]', re.IGNORECASE, ) # Matches a revert commit subject: `Revert "..."`, `Revert: ...`, or -# `revert(scope): ...`, case-insensitively. A revert's story ids must never -# be attributed as shipped — see the reverted_commits handling in main(). +# `revert(scope): ...`. REVERT_PATTERN = re.compile(r'^(revert\s+"|revert:\s|revert\()', re.IGNORECASE) SHORTCUT_API_BASE = "https://api.app.shortcut.com/api/v3" @@ -187,13 +99,7 @@ def extract_story_ids(text): def extract_pr_number(text): - """Return the LAST `(#NNN)` reference in text, matching the merge-commit - convention where a revert's own re-merge ref trails any PR ref quoted - from the original (reverted) subject, e.g. `Revert "x (#123)" (#456)`. - - Falls back to the bare "Merge pull request #N from ..." form when no - parenthesized ref is present -- that's the only form a real (non-squash) - merge commit ever gets.""" + """Return the LAST `(#NNN)` reference (e.g. `Revert "x (#123)" (#456)`), falling back to the bare "Merge pull request #N" form.""" if not text: return None matches = PR_PATTERN.findall(text) @@ -204,8 +110,7 @@ def extract_pr_number(text): def split_range_spec(spec): - """Split a PREV..CUR range string. Git refnames may never contain '..', - so the first occurrence is an unambiguous split point.""" + """Split a PREV..CUR range string on the first '..' (git refnames never contain '..').""" if ".." not in spec: die(f"--range must be of the form PREV..CUR, got: {spec!r}") idx = spec.index("..") @@ -218,20 +123,10 @@ def split_range_spec(spec): def resolve_range_from_version(version, chart_version=None): """Resolve the (prev, cur) prod/* tag pair for --version V [--chart-version CV]. - A prod app version can have MULTIPLE prod/* tags -- one per chart-only - rollout of the same app version (e.g. prod/6.100.0-prod.1+chart.0.85.8- - prod.{1,2,3}). With --chart-version, the exact tag `prod/V+chart.CV` is - preferred; if it doesn't exist (e.g. Argo's chartVersion arg and the tag - suffix drifted out of sync), that's logged and we fall through to the - no-chart-version path below rather than dying -- a chart-version mismatch - must never be the thing that kills a healthy rollout's release notes. - - Without a matching --chart-version, and when more than one tag matches - `prod/V+*`, the newest by tag creation date is chosen (never dies) and - the choice is logged. All tags are read once via `git tag --list - 'prod/*' --sort=-creatordate`, so "newest" and "previous tag" both use - that same deterministic ordering -- prev is simply the tag immediately - after the chosen one in that list. + Tags are read once via `git tag --list 'prod/*' --sort=-creatordate` + (newest first); with no matching --chart-version and multiple tags for + the same version, the newest is chosen and prev is the tag immediately + after it in that same ordering. """ v = version[1:] if version.startswith("v") else version @@ -277,20 +172,14 @@ def resolve_range_from_version(version, chart_version=None): def get_commit_subjects(range_spec): - # Deliberately NOT --no-merges: a merge-commit PR's bare "Merge pull - # request #N from .../hotfix/sc-.../..." subject can be the ONLY place - # its PR number and story id appear in the whole range (no squashed - # "(#N)" form, no matching child commit) -- see MERGE_PR_PATTERN and the - # NOISE_PATTERN handling for the plain branch-sync merges this also lets - # through. + # Deliberately not --no-merges: a merge commit's subject can be the only + # place its PR number and story id appear in the whole range. out = run_git(["log", range_spec, "--pretty=format:%s"]) return [line for line in out.split("\n") if line.strip()] def tag_creator_date_iso(tag): - """ISO 8601 creation date of an annotated tag (`%(creatordate:iso-strict)`), - or None if `tag` doesn't resolve to a real tag ref (e.g. an explicit - --range endpoint that's a branch or SHA, not a prod/* tag).""" + """ISO 8601 creation date of a tag, or None if `tag` isn't a real tag ref.""" out = run_git(["for-each-ref", "--format=%(creatordate:iso-strict)", f"refs/tags/{tag}"]).strip() return out or None @@ -322,10 +211,8 @@ def fetch_branches(pr_numbers, repo, max_workers=8): try: pr_number, branch = future.result() except FileNotFoundError: - # `gh` itself isn't on PATH -- every other in-flight lookup - # will hit the exact same error, so fail fast with one clear - # message instead of an unhandled traceback (or N identical - # per-PR warnings). + # gh not on PATH; every other in-flight lookup would fail + # identically, so fail fast with one clear message. die("`gh` was not found on PATH. Install the GitHub CLI " "(https://cli.github.com/) or ensure it's available in " "this environment; PR branch name lookups cannot proceed without it.") @@ -360,37 +247,10 @@ def fetch_story(story_id, token): def fetch_story_by_pr_link(pr_number, token): - """Look up the single Shortcut story linked to a merged PR via - Shortcut's own PR<->story association (RC1 in the incident writeup): - `GET search/stories?query=pr:`. This is a fallback ONLY for a commit - whose subject and PR branch name both carried no sc-NNNNN id -- git text - is not the only place the link can live; a story can be attached to a PR - from the Shortcut UI with no story code ever appearing in the branch - name (verified case: a PR branched as - `feature/prod-rollout-slack-release-notes`, no story code at all, that - Shortcut still knew was linked to a real story). - - `branch:"..."` and `pull-request:N` search operators do NOT resolve this - -- only `pr:N` does -- so that's the only query shape used here. - - Adopts the id ONLY when the search returns EXACTLY one story AND that - story's OWN linked-PR entry for this exact PR number passes the same - four PR-level guards reconcile_deploy_ready.py's sweep applies (merged - / Sefaria-Project repo / target branch master / head branch not a - long-lived environment branch -- see shortcut_pr_guards.py). That - second check matters because a bare - `pr:` match only proves Shortcut linked SOME story to this PR - number -- not that this PR is real shipping evidence for it. A - promotion PR (head branch `master`/`preprod`/`prod`) resolves via this - same search to a real story just as readily as that story's actual - feature PR does; without re-checking the guards here, that promotion - PR would get silently adopted as if it were proof the story shipped - (verified live). More than one search result is ambiguous (which story - is "the" story for this PR?) and is a warn-and-skip, never a guess. - Any lookup failure (network, HTTP error, bad JSON) is also a - warn-and-skip, matching fetch_story's error posture immediately above - -- this must never abort the run. - """ + """Look up the Shortcut story linked to a merged PR via `GET search/stories?query=pr:` + (only `pr:N` resolves this; `branch:`/`pull-request:` do not). Adopts the id only when + the search returns exactly one story and that story's matching PR passes passes_pr_guards; + ambiguous results or lookup failures are a warn-and-skip.""" query = urllib.parse.quote(f"pr:{pr_number}") url = f"{SHORTCUT_API_BASE}/search/stories?query={query}" req = urllib.request.Request(url, headers={"Shortcut-Token": token, "Accept": "application/json"}) @@ -440,8 +300,7 @@ def fetch_story_by_pr_link(pr_number, token): def fetch_stories_by_pr(pr_numbers, token, max_workers=8): - """Batch-resolve fetch_story_by_pr_link across PRs concurrently, the - same pattern fetch_branches and hydrate_stories already use below.""" + """Batch-resolve fetch_story_by_pr_link across PRs concurrently.""" story_id_by_pr = {} if not pr_numbers: return story_id_by_pr @@ -494,10 +353,7 @@ def build_arg_parser(): def chart_version_from_tag(tag): - """Prod tags carry the chart version after a '+', e.g. - prod/6.111.0-prod.2+chart.0.87.5-prod.1 -> 0.87.5-prod.1. - Returns None for a tag that does not follow that shape (e.g. an - explicit --range against arbitrary refs).""" + """Extract the chart version from a prod tag's '+chart.X' suffix, e.g. prod/6.111.0-prod.2+chart.0.87.5-prod.1 -> 0.87.5-prod.1. None if absent.""" if not tag: return None m = CHART_IN_TAG.search(tag) @@ -532,40 +388,18 @@ def main(): branch_by_pr = fetch_branches(sorted(pr_numbers), args.repo) - # Read once, up front: used both for the RC1 PR-link fallback right - # below and for hydration later in main(). A single token means both a - # missing token and an unreachable Shortcut behave identically in both - # places -- degrade gracefully, never abort the run. + # Read once; used both for the PR-link fallback below and hydration later. token = os.environ.get("SHORTCUT_API_TOKEN") - # Resolve each commit's full story_ids (subject ∪ its PR branch name) - # once, up front -- both the per-commit `commits[]` output and the - # aggregate shipped-set logic below read from this. + # Resolve each commit's full story_ids (subject ∪ its PR branch name) once. for c in parsed_commits: branch = branch_by_pr.get(c["pr_number"]) if c["pr_number"] else None c["branch"] = branch c["story_ids"] = c["subject_story_ids"] | extract_story_ids(branch) - # THIRD discovery source (RC1): for any commit that has a PR number but - # STILL resolved to no story id from subject+branch, ask Shortcut - # itself whether that PR is linked to a story. Done at this same - # resolution point -- before carrying_indices_by_id, reverted_commits, - # commits_without_story, or the aggregate shipped set are computed -- - # so every one of those downstream consumers sees the recovered id as - # if it had always been there, with no separate code path to keep in - # sync. - # - # A commit is eligible for this lookup only if, in addition to "has a PR - # number but no story id yet": its subject isn't auto-generated - # merge/branch-sync noise (NOISE_PATTERN already excludes exactly this - # shape from commits_without_story for the same reason -- reused here - # rather than inventing a second notion of "not a real feature commit"), - # and its PR's own head branch isn't a long-lived environment branch - # (LONG_LIVED_ENV_BRANCHES). Both are shapes a promotion PR takes, and a - # promotion PR resolves via `pr:` to a real story just as readily as - # that story's actual feature PR does -- verified live -- so both are - # filtered out here, BEFORE ever calling Shortcut, rather than relying - # solely on fetch_story_by_pr_link's own re-check of the PR itself. + # Third discovery source: for a commit with a PR number but no story id + # yet, ask Shortcut whether that PR is linked to a story -- unless the + # subject is noise or the PR's head branch is a promotion merge. def _eligible_for_pr_link_fallback(c): return ( c["pr_number"] @@ -595,11 +429,9 @@ def _eligible_for_pr_link_fallback(c): "no sc-NNNNN id in its subject or branch name." ) - # Track, per story id, which NON-revert commit indices carry it. This is - # what lets a revert exclude ONLY the specific original commit it quotes - # from the shipped set -- not every commit that happens to share that id - # -- so an id independently carried by another, still-live commit keeps - # shipping (finding #7). + # Tracks which non-revert commits carry each story id, so a revert + # excludes only the specific commit it quotes, not every commit sharing + # that id. carrying_indices_by_id = {} for i, c in enumerate(parsed_commits): if c["is_revert"]: @@ -612,17 +444,15 @@ def _eligible_for_pr_link_fallback(c): if not c["is_revert"]: continue - # A revert's OWN story ids must never be attributed as shipped, but - # they're too important to silently drop — surface them instead. + # A revert's own story ids are never attributed as shipped, but are surfaced separately. if c["story_ids"]: reverted_commits.append({ "subject": c["subject"], "suppressed_story_ids": sorted(c["story_ids"], key=int), }) - # If this revert quotes a commit subject that's ALSO in this same - # range, that original's ids must stop shipping too -- unless some - # other, still-live commit independently carries the same id. + # If the reverted commit is also in this range, stop its ids from + # shipping too, unless another commit still carries them. quote_match = REVERT_QUOTE_PATTERN.match(c["subject"]) if not quote_match: continue @@ -645,9 +475,6 @@ def _eligible_for_pr_link_fallback(c): "story_ids": sorted(c["story_ids"], key=int), }) - # Reverts are excluded here too: a revert with a suppressed story id - # is already reported via reverted_commits, and a revert with no - # story id at all is ordinary noise, not a commit needing attention. if not c["is_revert"] and not c["story_ids"] and not NOISE_PATTERN.search(c["subject"]): commits_without_story.append(c["subject"]) @@ -662,26 +489,16 @@ def _eligible_for_pr_link_fallback(c): result = { "version": version, "chart_version": chart_version_from_tag(cur), - # ISO 8601 creation date of the resolved current tag, or null if - # `cur` isn't a real tag ref (e.g. an explicit --range against a - # branch/SHA). Source of truth for the release date shown in the - # generated Slack posts -- see sefaria-release-notes/SKILL.md. "release_date": tag_creator_date_iso(cur), "range": {"previous_tag": prev, "current_tag": cur, "spec": range_spec}, "commits": commits, "commits_without_story": commits_without_story, "reverted_commits": reverted_commits, "story_ids": sorted(all_story_ids, key=int), - # Ids adopted ONLY via the RC1 Shortcut PR-link fallback above -- - # i.e. story ids git text alone (subject + branch name) never - # revealed. Every id here is also already included in "story_ids" - # (and, if hydration succeeded, in "stories"); this list exists so - # a report can call out what only Shortcut knew. + # Ids adopted only via the PR-link fallback; already included in story_ids/stories. "stories_from_shortcut_pr_link": sorted(stories_from_shortcut_pr_link, key=int), "stories": stories, - # False when SHORTCUT_API_TOKEN was absent, so `stories` being empty - # means "never looked up" rather than "looked up and found nothing". - # Without this the two cases are indistinguishable downstream. + # False when SHORTCUT_API_TOKEN was absent, distinguishing "never looked up" from "found nothing". "hydrated": hydrated, "unresolved_story_ids": unresolved_story_ids, } diff --git a/build/ci/shortcut_pr_guards.py b/build/ci/shortcut_pr_guards.py index 5e27e771c5..2a37d0f6d5 100644 --- a/build/ci/shortcut_pr_guards.py +++ b/build/ci/shortcut_pr_guards.py @@ -1,79 +1,18 @@ #!/usr/bin/env python3 """ -Shared "does this linked PR count as shipping evidence" guards. - -Both `shipped_stories.py` (RC1's PR<->story link fallback) and -`reconcile_deploy_ready.py` (RC2's org-wide Deploy Ready sweep) need to -answer the exact same question about a PR that Shortcut says is linked to a -story: does this PR actually prove that story's change reached prod? A -linked PR is NOT automatically that evidence -- FOUR guards apply, and -both scripts must apply the SAME four guards or they will silently drift -apart. They already have, twice, in opposite directions: RC1's PR-link -fallback initially had no guards at all (a promotion PR resolved via -`search/stories?query=pr:` to a real story just as readily as that -story's actual feature PR); then, after guards 1-3 were added here, a -promotion PR merging INTO master (rather than out of it) turned out to -still pass all three -- verified live: a real story was classified -`shipped` on the strength of a PR whose head branch was `preprod` and -target branch was `master`, a promotion merge, while its genuine feature -PR (head a hotfix/bugfix branch, target a hotfix branch) was correctly -rejected by guard 3 for not targeting `master` directly. Guard 3 alone -cannot catch this shape: a promotion merge legitimately targets `master`, -so the target-branch check has nothing to object to. The giveaway is the -SOURCE (head) branch, not the target -- hence guard 4. - - 1. `merged` must be true. An open or closed-without-merging PR is not - evidence anything shipped. - 2. `repository_id` must be Sefaria-Project's own (500000103). A story can - link a PR from a different repo; resolving that PR number against - Sefaria-Project instead would find an unrelated (often much older) PR - that happens to share the number. - 3. `target_branch_name` must be "master". A promotion PR (preprod -> - prod, or master -> preprod) merges constantly and proves nothing - about whether a given story's own change reached prod -- it must - never be treated as interchangeable with the real feature PR. - 4. `branch_name` (the PR's HEAD/source branch) must NOT be a long-lived - environment branch (master, preprod, prod). A promotion PR that - merges ONE of those branches INTO master (e.g. preprod -> master, the - opposite direction from guard 3's preprod/prod targets) passes guards - 1-3 cleanly -- it's merged, against the right repo, and its target - really is "master". Only the head branch reveals it's a promotion - merge, not a feature PR. - -This module holds the shared implementation so there is exactly one place -these guards live; single-source-of-truth, not two parallel copies that a -future edit only remembers to update in one of them. Stdlib only -- no -third-party dependencies, matching both callers' dependency posture. - -All story/PR ids in this file's docstring and comments (e.g. 500000103, -which is Sefaria-Project's real repository id, not a story id, and is not -covered by the "no real Shortcut ids" convention the two callers document) -are either placeholders or non-story ids -- see shipped_stories.py's and -reconcile_deploy_ready.py's own docstrings for that convention. +Shared guards for deciding whether a Shortcut-linked PR counts as evidence +that a story's change reached prod. """ -# Sefaria-Project's own Shortcut repository id. Not a story id -- see the -# module docstring's placeholder-id note. SEFARIA_PROJECT_REPO_ID = 500000103 DEFAULT_TARGET_BRANCH = "master" -# Guard #4. Shared with shipped_stories.py's own pre-filter (it also skips -# the RC1 fallback lookup entirely for a commit whose PR head branch is one -# of these -- see that script's LONG_LIVED_ENV_BRANCHES, which now imports -# this same set rather than keeping a second copy) -- the two must name the -# exact same branches or they can drift apart on what counts as "long-lived" -# the same way they already drifted on whether this guard existed at all. LONG_LIVED_ENV_BRANCHES = frozenset({"master", "preprod", "prod"}) def passes_pr_guards(pr, repo_id=SEFARIA_PROJECT_REPO_ID, target_branch=DEFAULT_TARGET_BRANCH): - """True if a single linked-PR object (a Shortcut `pull-request` entity, - as found in a story's `pull_requests` or `branches[*].pull_requests`) - counts as evidence that a story's change reached prod: merged, against - the right repo, targeting the right branch, and NOT itself a promotion - merge (head branch not long-lived). See the module docstring for why - each of the four checks exists.""" + """True if a linked-PR object counts as shipping evidence: merged, right repo, right target branch, and not a promotion merge.""" return ( pr.get("merged") is True and pr.get("repository_id") == repo_id @@ -83,20 +22,12 @@ def passes_pr_guards(pr, repo_id=SEFARIA_PROJECT_REPO_ID, target_branch=DEFAULT_ def qualifying_prs(prs, repo_id=SEFARIA_PROJECT_REPO_ID, target_branch=DEFAULT_TARGET_BRANCH): - """Filter a list of linked-PR objects down to the ones that pass - passes_pr_guards.""" + """Filter a list of linked-PR objects down to the ones that pass passes_pr_guards.""" return [pr for pr in prs if passes_pr_guards(pr, repo_id=repo_id, target_branch=target_branch)] def gather_linked_prs(story): - """Collect every PR linked to a Shortcut story from BOTH - `story.pull_requests` and `story.branches[*].pull_requests` -- Shortcut - duplicates the same PR object in both places, and depending on how/when - a branch or PR was linked, either one can be the only place a given PR - shows up. Deduplicated by PR number (top-level `pull_requests` wins on - a tie; the object is identical either way). Returns every linked PR, - guards not yet applied -- see qualifying_prs/passes_pr_guards -- so a - caller can show what was linked at all, not just what passed.""" + """Collect every PR linked to a story from both `pull_requests` and `branches[*].pull_requests`, deduplicated by PR number.""" seen = set() prs = [] for pr in story.get("pull_requests") or []: diff --git a/build/ci/triage_explainer.py b/build/ci/triage_explainer.py index b17098f2bc..c4bce63873 100644 --- a/build/ci/triage_explainer.py +++ b/build/ci/triage_explainer.py @@ -1,62 +1,19 @@ #!/usr/bin/env python3 """ -Extract JUST the triage bucket from a reconcile_deploy_ready.py report, -plus the minimal safe context an explainer needs, into a separate, -structurally isolated document. - -Today reconcile_deploy_ready.py's triage bucket is reported as -`reason=no_qualifying_pr` (or `non_standard_workflow_or_state`) plus raw -diagnostic fields, which leaves a human to open every story and work out -each one individually. An OPT-IN workflow step -(.github/workflows/prod-release-notes.yaml) runs a headless `claude -p` -over exactly the document this script produces to propose a short -hypothesis and a suggested next action per triage story -- but the LLM -call itself lives entirely in that workflow step, never here. This script -is a plain, deterministic JSON transform: no API client, no `claude` -invocation, no network. Keeping reconcile_deploy_ready.py (and this -sibling script) stdlib-only and LLM-free, with the model's role confined -to writing English from a document it's handed, is the same design -principle the rest of this pipeline already follows (see -reconcile_deploy_ready.py's and shipped_stories.py's own docstrings). - -Why a SEPARATE document rather than just telling the model "only look at -the triage section" of the full report: the full report's `shipped` and -`pending` buckets, and reconcile_deploy_ready.py's own JSON write-back -comment content, must never be visible to, scored by, or able to -influence this explainer -- and a prompt instruction is not a security -boundary against a prompt-injection path. A triage story's `description` -and `comments` fields are CONTRIBUTOR-CONTROLLED TEXT (see the workflow -step's own comment for why that scopes its allowed tools). Rather than -trust the model to honor "ignore the other buckets" against adversarial -input embedded in the very document it's reading, this script simply never -puts shipped/pending data into the file the explainer is given at all -- -there is nothing there to leak or be steered by, structurally, not merely -by convention. - -This script also owns the opt-in DECISION (resolve_enabled / -`resolve-enabled` subcommand) for the same single-source-of-truth reason: -without it, "is the explainer enabled" would be a small bash string -comparison duplicated (and possibly drifted) inline in the workflow YAML, -untested by anything. Putting it here means the workflow's `run:` block -just calls this script and the actual rule lives in one tested place. +Extracts the triage bucket from a reconcile_deploy_ready.py report into a +separate, isolated document, and resolves whether the opt-in triage +explainer should run. Usage: python3 triage_explainer.py extract --report reconcile-deploy-ready-report.json --out triage-only.json python3 triage_explainer.py resolve-enabled --event-name workflow_dispatch --explain-triage-input true --enable-var "" - -All story ids in this file's docstring and comments (e.g. 11111) are -placeholders, not real Shortcut story ids. """ import argparse import json import sys -# Keys from a reconcile_deploy_ready.py report that this script MUST NEVER -# copy into its output, even if a future edit to that report adds new -# top-level keys carelessly. Listed explicitly (rather than an -# allow-only-"triage" approach implemented by construction below) as a -# second, redundant line of defense -- see extract_triage_only. +# Must never appear in the explainer's output. EXCLUDED_REPORT_KEYS = frozenset({"shipped", "pending", "applied"}) @@ -66,27 +23,14 @@ def die(message: str) -> None: def extract_triage_only(report): - """Build the explainer's ENTIRE input: the triage list verbatim, as - reconcile_deploy_ready.py already enriched each entry (description, - comments, linked_prs with per-PR guard diagnostics -- see - reconcile_deploy_ready.classify_stories / _triage_context / - _diagnose_linked_pr), plus `prod_tag` (so an explanation can say "as of - " without guessing) and a `triage_count` the caller can check - cheaply to skip the whole explainer step when there's nothing to - explain. This is constructed as an explicit allow-list (only these - three keys are ever read from `report` and copied out) rather than - "copy everything except EXCLUDED_REPORT_KEYS" -- an allow-list can't - accidentally leak a new field a future report format adds; a - deny-list could.""" + """Build the explainer's entire input via an explicit allow-list: triage list, prod_tag, and triage_count.""" triage = report.get("triage") or [] result = { "prod_tag": report.get("prod_tag"), "triage_count": len(triage), "triage": triage, } - # Defensive, should be unreachable given the allow-list above -- kept - # as a loud assertion rather than silently trusting the allow-list - # forever stays correct. + # Defensive: should be unreachable given the allow-list above. leaked = EXCLUDED_REPORT_KEYS & result.keys() if leaked: die(f"internal error: triage-only extraction would have leaked {sorted(leaked)} -- refusing to write it") @@ -94,23 +38,7 @@ def extract_triage_only(report): def resolve_enabled(event_name, explain_triage_input, enable_var): - """Whether the opt-in triage explainer should run this trigger. Pure - decision logic, no I/O -- both inputs default to disabled on any falsy - or unrecognized value, never enabled by omission or by an unexpected - string: - - - A `workflow_dispatch` run opts in per-run via its own - `explain_triage` boolean input. - - The `repository_dispatch` trigger (the real automatic - post-promotion path -- see the workflow header) carries no such - input at all, so it instead opts in via a repo-level Actions - variable (`vars.ENABLE_TRIAGE_EXPLAINER`), which a human sets - independently of any single run. - - Only an exact case-insensitive "true" enables anything; every other - value (empty, "false", "1", a typo, ...) is treated as disabled. This - means the feature ships fully OFF by default on every trigger path - until a human explicitly flips one of the two switches.""" + """Whether the opt-in triage explainer should run: workflow_dispatch opts in via its own input, other triggers via the ENABLE_TRIAGE_EXPLAINER repo variable.""" value = explain_triage_input if event_name == "workflow_dispatch" else enable_var return str(value).strip().lower() == "true" From b79f8225e58d6033a59d2dbcd84fbfa6ac64274c Mon Sep 17 00:00:00 2001 From: Yotam Fromm Date: Mon, 7 Sep 2026 11:26:41 +0300 Subject: [PATCH 6/6] fix(ci): 6 review findings on the reconcile pipeline 1. Pass --prod-tag (from shipped-stories.json's range.current_tag) to reconcile_deploy_ready.py in the workflow -- it was silently falling back to the newest prod/* tag on every run, including a workflow_dispatch re-run for an older release, which would wrongly gate hydrated_story against the wrong tag. 2. reconcile_deploy_ready.py now warns and exits non-zero (under --apply only) when it saw stories, transitioned none, and routed some to triage -- mirrors mark_stories_deployed.py's existing silent-no-op guard. 3. Scope the triage-explainer Claude step's cwd to $RUNNER_TEMP -- its Write grant previously covered the checkout, including shipped-stories.json, while processing contributor-controlled text. 4. Gate the RC1 PR-link fallback on `not c["is_revert"]` -- a revert's own merge-commit PR could otherwise get adopted into stories_from_shortcut_pr_link despite never reaching story_ids. 5. Add steps.merge_backfill.outcome == 'failure' to both bookkeeping warn/alert step conditions -- a merge failure was invisible. 6. Pin the Claude Code CLI install to @2.1.263 (the version currently resolving) instead of floating on latest. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01KJqBksQqzHYs3F54Y4tJRX --- .github/workflows/prod-release-notes.yaml | 50 ++++++++++++------- build/ci/reconcile_deploy_ready.py | 11 ++++ build/ci/shipped_stories.py | 1 + build/ci/tests/test_reconcile_deploy_ready.py | 39 +++++++++++++++ build/ci/tests/test_shipped_stories.py | 31 ++++++++++++ 5 files changed, 113 insertions(+), 19 deletions(-) diff --git a/.github/workflows/prod-release-notes.yaml b/.github/workflows/prod-release-notes.yaml index ca94a99582..a62e34fcbc 100644 --- a/.github/workflows/prod-release-notes.yaml +++ b/.github/workflows/prod-release-notes.yaml @@ -56,7 +56,7 @@ jobs: node-version: "20" - name: Install Claude Code CLI - run: npm install -g @anthropic-ai/claude-code + run: npm install -g @anthropic-ai/claude-code@2.1.263 - name: Resolve version id: resolve @@ -125,17 +125,33 @@ jobs: EVENT_NAME: ${{ github.event_name }} DRY_RUN_INPUT: ${{ inputs.dry_run }} run: | - ARGS=(--out "$RUNNER_TEMP/reconcile-deploy-ready-report.json") + CURRENT_TAG=$(python3 -c "import json; print(json.load(open('shipped-stories.json'))['range']['current_tag'])") + ARGS=(--out "$RUNNER_TEMP/reconcile-deploy-ready-report.json" --prod-tag "$CURRENT_TAG") if [[ "$EVENT_NAME" != "workflow_dispatch" || "$DRY_RUN_INPUT" != "true" ]]; then ARGS+=(--apply) fi python3 build/ci/reconcile_deploy_ready.py "${ARGS[@]}" + - name: Merge current-release backfill into shipped-stories.json + id: merge_backfill + # Must run after "Reconcile Deploy Ready backlog" and before "Generate release notes". + continue-on-error: true + run: | + REPORT_FILE="$RUNNER_TEMP/reconcile-deploy-ready-report.json" + if [[ ! -s "$REPORT_FILE" ]]; then + echo "No reconcile report found at $REPORT_FILE (the reconcile step may have failed or produced nothing) -- nothing to merge." + exit 0 + fi + python3 build/ci/merge_release_backfill.py \ + --shipped-stories-out shipped-stories.json \ + --reconcile-report "$REPORT_FILE" + - name: Warn if Shortcut bookkeeping failed - if: steps.mark_deployed.outcome == 'failure' || steps.reconcile.outcome == 'failure' + if: steps.mark_deployed.outcome == 'failure' || steps.reconcile.outcome == 'failure' || steps.merge_backfill.outcome == 'failure' env: MARK_DEPLOYED_OUTCOME: ${{ steps.mark_deployed.outcome }} RECONCILE_OUTCOME: ${{ steps.reconcile.outcome }} + MERGE_BACKFILL_OUTCOME: ${{ steps.merge_backfill.outcome }} run: | if [[ "$MARK_DEPLOYED_OUTCOME" == "failure" ]]; then echo "::warning::mark_stories_deployed.py failed -- shipped Shortcut stories were NOT moved Deploy Ready -> Done for this release. Release notes generation is continuing anyway. Check this run's 'Mark shipped stories as deployed' step and move the affected stories manually." @@ -143,14 +159,18 @@ jobs: if [[ "$RECONCILE_OUTCOME" == "failure" ]]; then echo "::warning::reconcile_deploy_ready.py failed -- the org-wide Deploy Ready backlog sweep did not complete. Release notes generation is continuing anyway. Check this run's 'Reconcile Deploy Ready backlog' step; the backlog will be retried on the next release." fi + if [[ "$MERGE_BACKFILL_OUTCOME" == "failure" ]]; then + echo "::warning::merge_release_backfill.py failed -- a backfilled story that shipped in this release may be missing from the announcement. Check this run's 'Merge current-release backfill into shipped-stories.json' step." + fi - name: Alert Slack if Shortcut bookkeeping failed - if: steps.mark_deployed.outcome == 'failure' || steps.reconcile.outcome == 'failure' + if: steps.mark_deployed.outcome == 'failure' || steps.reconcile.outcome == 'failure' || steps.merge_backfill.outcome == 'failure' env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEPLOY_WEBHOOK }} VERSION: ${{ steps.resolve.outputs.version }} MARK_DEPLOYED_OUTCOME: ${{ steps.mark_deployed.outcome }} RECONCILE_OUTCOME: ${{ steps.reconcile.outcome }} + MERGE_BACKFILL_OUTCOME: ${{ steps.merge_backfill.outcome }} run: | FAILURES="" if [[ "$MARK_DEPLOYED_OUTCOME" == "failure" ]]; then @@ -159,24 +179,13 @@ jobs: if [[ "$RECONCILE_OUTCOME" == "failure" ]]; then FAILURES="${FAILURES}reconcile_deploy_ready.py failed (org-wide Deploy Ready backlog sweep did not complete). " fi + if [[ "$MERGE_BACKFILL_OUTCOME" == "failure" ]]; then + FAILURES="${FAILURES}merge_release_backfill.py failed (a backfilled story may be missing from the announcement). " + fi curl -s -X POST -H 'Content-type: application/json' \ --data "{\"text\": \":warning: Prod release notes for version ${VERSION:-unknown}: ${FAILURES}Release notes generation is continuing. Check the workflow run and move the affected stories manually.\"}" \ "$SLACK_WEBHOOK_URL" || true - - name: Merge current-release backfill into shipped-stories.json - id: merge_backfill - # Must run after "Reconcile Deploy Ready backlog" and before "Generate release notes". - continue-on-error: true - run: | - REPORT_FILE="$RUNNER_TEMP/reconcile-deploy-ready-report.json" - if [[ ! -s "$REPORT_FILE" ]]; then - echo "No reconcile report found at $REPORT_FILE (the reconcile step may have failed or produced nothing) -- nothing to merge." - exit 0 - fi - python3 build/ci/merge_release_backfill.py \ - --shipped-stories-out shipped-stories.json \ - --reconcile-report "$REPORT_FILE" - - name: Resolve triage explainer opt-in id: triage_opt_in env: @@ -196,6 +205,9 @@ jobs: if: steps.triage_opt_in.outputs.enabled == 'true' # Opt-in and best-effort; never blocks the release announcement. continue-on-error: true + # Input is contributor-controlled text; cwd here (not the checkout) + # is what actually bounds the step's Write grant. + working-directory: ${{ runner.temp }} env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: | @@ -211,7 +223,7 @@ jobs: fi TRIAGE_ONLY_FILE="$RUNNER_TEMP/reconcile-triage-only.json" - python3 build/ci/triage_explainer.py extract --report "$REPORT_FILE" --out "$TRIAGE_ONLY_FILE" + python3 "$GITHUB_WORKSPACE/build/ci/triage_explainer.py" extract --report "$REPORT_FILE" --out "$TRIAGE_ONLY_FILE" TRIAGE_COUNT=$(python3 -c "import json; print(json.load(open('$TRIAGE_ONLY_FILE'))['triage_count'])") if [[ "$TRIAGE_COUNT" == "0" ]]; then diff --git a/build/ci/reconcile_deploy_ready.py b/build/ci/reconcile_deploy_ready.py index f157ceff5b..c707414068 100644 --- a/build/ci/reconcile_deploy_ready.py +++ b/build/ci/reconcile_deploy_ready.py @@ -459,6 +459,17 @@ def main(): ) sys.exit(1) + # Nothing shipped but stories exist and some routed to triage: could be + # legitimate, or a guard misconfiguration (e.g. SEFARIA_PROJECT_REPO_ID + # drift) silently routing everything to triage forever. --dry-run is a + # preview by definition and never exits non-zero for this. + if apply_mutations and stories and not shipped and triage: + warn( + f"{len(stories)} Deploy Ready stor{'y' if len(stories) == 1 else 'ies'} seen, 0 shipped, " + f"{len(triage)} routed to triage. Check for guard drift before assuming the backlog is clean." + ) + sys.exit(2) + if __name__ == "__main__": main() diff --git a/build/ci/shipped_stories.py b/build/ci/shipped_stories.py index a0344424eb..fb5343dc6b 100755 --- a/build/ci/shipped_stories.py +++ b/build/ci/shipped_stories.py @@ -404,6 +404,7 @@ def _eligible_for_pr_link_fallback(c): return ( c["pr_number"] and not c["story_ids"] + and not c["is_revert"] and not NOISE_PATTERN.search(c["subject"]) and c["branch"] not in LONG_LIVED_ENV_BRANCHES ) diff --git a/build/ci/tests/test_reconcile_deploy_ready.py b/build/ci/tests/test_reconcile_deploy_ready.py index 0529efc02e..a4dd44ef14 100644 --- a/build/ci/tests/test_reconcile_deploy_ready.py +++ b/build/ci/tests/test_reconcile_deploy_ready.py @@ -694,6 +694,31 @@ def test_main_failed_transition_exits_non_zero_and_is_reported(monkeypatch, tmp_ assert "11111" in err +def test_main_apply_all_triage_no_shipped_warns_and_exits_nonzero(monkeypatch, tmp_path, capsys): + """Every story routes to triage, none shipped -- must not exit 0 as if + the backlog were clean.""" + story = _story(11111) # no linked PRs -> triage + _make_main_env(monkeypatch, tmp_path, [story], argv_extra=["--apply"]) + + with pytest.raises(SystemExit) as exc_info: + rdr.main() + assert exc_info.value.code != 0 + assert "WARNING" in capsys.readouterr().err + + +def test_main_dry_run_all_triage_no_shipped_does_not_exit_nonzero(monkeypatch, tmp_path): + story = _story(11111) + _make_main_env(monkeypatch, tmp_path, [story]) # dry-run default, no --apply + + rdr.main() # must not raise + + +def test_main_apply_all_triage_but_no_stories_at_all_does_not_exit_nonzero(monkeypatch, tmp_path): + """Empty backlog is not a silent no-op -- there's nothing to be silent about.""" + _make_main_env(monkeypatch, tmp_path, [], argv_extra=["--apply"]) + rdr.main() # must not raise + + def test_main_prod_tag_override_is_used_instead_of_default(monkeypatch, tmp_path): story = _story(11111, pull_requests=[_pr(3606)]) out_path = tmp_path / "report.json" @@ -717,6 +742,20 @@ def _boom_default_tag(): assert report["counts"]["shipped"] == 1 +def test_workflow_passes_prod_tag_to_reconcile_step(): + """reconcile_deploy_ready.py's own --prod-tag handling is correct + (tested above), but nothing at that level can catch the workflow + itself forgetting to pass the flag -- assert the step's run block + actually does.""" + import pathlib + + workflow = pathlib.Path(__file__).resolve().parents[3] / ".github" / "workflows" / "prod-release-notes.yaml" + text = workflow.read_text(encoding="utf-8") + start = text.index("Reconcile Deploy Ready backlog") + end = text.index("- name:", start + 1) + assert "--prod-tag" in text[start:end] + + # --- resolve_default_prod_tag: newest by creation date ------------------- def test_resolve_default_prod_tag_picks_newest(monkeypatch): diff --git a/build/ci/tests/test_shipped_stories.py b/build/ci/tests/test_shipped_stories.py index 43144762b5..0f1a8c3f9f 100644 --- a/build/ci/tests/test_shipped_stories.py +++ b/build/ci/tests/test_shipped_stories.py @@ -887,6 +887,37 @@ def _boom(*args, **kwargs): assert data["story_ids"] == [] +def test_main_pr_link_fallback_skips_revert_commits(monkeypatch, tmp_path): + """A revert's own merge-commit PR must never trigger the fallback -- + its story ids are suppressed from the shipped set regardless, so a + recovered id here would appear in stories_from_shortcut_pr_link while + never actually being in story_ids/stories.""" + + def _fake_run_git(args): + if args[0] == "log": + return 'Revert "feat: something (#123)" (#456)\n' + if args[0] == "for-each-ref": + return "" + raise AssertionError(f"unexpected git call: {args!r}") + + def _boom(*args, **kwargs): + raise AssertionError("urlopen must never be called for a revert commit's PR") + + monkeypatch.setattr(ss, "run_git", _fake_run_git) + monkeypatch.setattr(ss, "fetch_pr_branch", lambda pr_number, repo: (pr_number, "feature/some-branch")) + monkeypatch.setenv("SHORTCUT_API_TOKEN", "fake-token-for-tests") + monkeypatch.setattr(ss.urllib.request, "urlopen", _boom) + + out_path = tmp_path / "shipped-stories.json" + monkeypatch.setattr( + "sys.argv", + ["shipped_stories.py", "--range", "prev-tag..cur-tag", "--out", str(out_path)], + ) + ss.main() + data = json.loads(out_path.read_text(encoding="utf-8")) + assert data["stories_from_shortcut_pr_link"] == [] + + def test_main_pr_link_fallback_not_triggered_when_subject_already_has_story_id(monkeypatch, tmp_path): """A commit whose subject already carries a story id must never trigger the fallback lookup at all -- it has nothing missing to recover."""