From dca874b3bcf310a71d9cb773927d185d689be11c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:15:24 +0900 Subject: [PATCH 1/3] fix(scheduler): resolve live refs before cancelling runs --- .../workflows/pr-review-merge-scheduler.yml | 45 ++++++++++++------- CHANGELOG.md | 4 ++ docs/doctoring/queue-hygiene-live-ref-race.md | 37 +++++++++++++++ docs/product-technical-gap-baseline.md | 1 + .../test_required_workflow_queue_contract.py | 4 ++ 5 files changed, 75 insertions(+), 16 deletions(-) create mode 100644 docs/doctoring/queue-hygiene-live-ref-race.md diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index a9bb54f8a..881165240 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -1068,29 +1068,42 @@ jobs: # Compare live refs on every sweep instead of waiting for an age # threshold: previous-head checks are never useful merge evidence. queue_hygiene_ready=true - if ! open_pr_heads_json="$( + open_pr_heads_json="{}" + if open_pr_refs_tsv="$( gh api \ -H "Accept: application/vnd.github+json" \ "/repos/${repo_full_name}/pulls?state=open&per_page=100" \ --paginate \ - | jq -sc ' - add - | map( - select( - .head.repo.full_name != null and - .head.ref != null and - .head.sha != null - ) - | { - key: "\(.head.repo.full_name):\(.head.ref)", - value: .head.sha - } - ) - | from_entries + | jq -sr ' + add[] + | select(.head.repo.full_name != null and .head.ref != null) + | [.head.repo.full_name, .head.ref] + | @tsv ' )"; then + while IFS=$'\t' read -r head_repo head_ref; do + [ -n "$head_repo" ] && [ -n "$head_ref" ] || continue + encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | @uri')" + if ! live_head_sha="$( + gh api \ + -H "Accept: application/vnd.github+json" \ + "/repos/${head_repo}/git/ref/heads/${encoded_head_ref}" \ + --jq '.object.sha // empty' + )" || ! [[ "$live_head_sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: live ref ${head_repo}:${head_ref} could not be resolved safely. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + break + fi + open_pr_heads_json="$( + jq \ + --arg key "${head_repo}:${head_ref}" \ + --arg value "$live_head_sha" \ + '. + {($key): $value}' \ + <<<"$open_pr_heads_json" + )" + done <<<"$open_pr_refs_tsv" + else echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: open PR head refs could not be read safely. No run will be cancelled from incomplete evidence." - open_pr_heads_json="{}" queue_hygiene_ready=false fi if ! current_default_sha="$( diff --git a/CHANGELOG.md b/CHANGELOG.md index cef0acda6..80eb22873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Resolve each open PR head through its live Git reference before queue hygiene + cancels superseded runs, and fail closed when any ref cannot be read. This + prevents a briefly stale pull-request payload from cancelling current-head + Checks without adding an arbitrary grace period. - Route Strix cross-provider fallbacks to explicit direct-OpenAI models (`openai-direct/...`) through the OpenAI inference endpoint instead of inheriting a provider-specific primary base: the workflow now provisions diff --git a/docs/doctoring/queue-hygiene-live-ref-race.md b/docs/doctoring/queue-hygiene-live-ref-race.md new file mode 100644 index 000000000..775ca39d8 --- /dev/null +++ b/docs/doctoring/queue-hygiene-live-ref-race.md @@ -0,0 +1,37 @@ +# Queue hygiene live-reference race + +## Incident + +On 2026-08-26, LineageWeave PR #667 received a new same-repository head +`37cc9ab1163f213105d420618e2e8ee69ec6673d`. Its new pull-request workflows +started, but the organization queue sweep cancelled them while GitHub's open-PR +payload still exposed the preceding head. The runs were current for the branch +ref and stale only in the pull-request listing used by the cancellation map. + +This was a control-plane defect, not a test failure. Re-running the jobs without +repairing the comparison source would leave the same race available to every +repository in the organization. + +## Decision + +Queue hygiene still enumerates open pull requests to identify eligible head +repositories and branch names. Before cancelling anything, it now resolves each +head through GitHub's `Get a reference` endpoint and compares active runs with +that live Git reference. A missing, inaccessible, or malformed ref makes the +repository's cancellation pass unavailable; no run is cancelled from partial +evidence. + +No time delay or grace-period heuristic is used. A branch ref is the exact +commit pointer the check run is meant to validate. The existing rule remains: +previous-head runs may be cancelled, current-head runs may not. + +## Verification + +- `uv run --group dev pytest -q tests/test_required_workflow_queue_contract.py` +- `actionlint .github/workflows/pr-review-merge-scheduler.yml` +- `git diff --check` + +## Reference + +GitHub. (n.d.). *REST API endpoints for Git references*. GitHub Docs. Retrieved +August 26, 2026, from https://docs.github.com/en/rest/git/refs diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 252249095..7f797d9e6 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -88,6 +88,7 @@ flowchart LR | G-12 | CSAP/SOC 2 통제 목표와 PII masking 대안은 doctoring에 흩어져 있으며 evidence-to-control mapping의 live completeness가 미확인이다 | PII를 마스킹하면 업무가 멈추고, 원문 접근을 허용하면 감사·유출 위험이 커진다 | consent/purpose/access lease, field-level encryption/tokenization, redaction-at-egress, audit/revocation와 CSAP/SOC 2 evidence map을 구현한다 | | G-13 | hourly scheduler는 존재하지만 no-op/credential unavailable/queued Checks의 customer next action을 모든 caller가 동일한 receipt로 내는지 미확인이다 | 자동화가 실패해도 운영자가 무엇을 고쳐야 하는지 알 수 없다 | `skipped_credential_unavailable` receipt와 다음 행동 문구를 exact-head Checks로 검증한 뒤 병합하고, bounded receipt schema, retry floor, single-flight, no secret fallback을 모든 caller contract test로 고정한다 | | G-14 | release/changelog/version 증거가 각 PR에 분산되고 현재 central repo 보호 main의 release candidate가 명확하지 않다 | 운영자는 어떤 기능이 supportable release인지 확인할 수 없다 | merge 후 release readiness ledger, CHANGELOG, semantic version/tag, rollback/operability evidence를 함께 갱신한다 | +| G-15 | 조직 큐 정리기가 open-PR payload의 직전 head와 새 branch ref가 잠시 달랐던 LineageWeave #667의 current-head runs를 취소했다 | 정상 Checks가 코드 실패처럼 사라지고 모든 소비 저장소의 병합 루프가 반복 재실행에 갇힌다 | open PR의 repo/ref는 탐색에만 쓰고 취소 판정 SHA는 live Git reference에서 다시 읽는다. ref를 완전하게 읽지 못하면 해당 저장소에서는 아무 run도 취소하지 않는다 | ## 4. 열린 PR live inventory diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 1d79f1daa..67c2fe69b 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -748,6 +748,10 @@ def test_org_queue_sweep_covers_target_repositories_on_a_heartbeat() -> None: assert "ORG_SWEEP_STALE_QUEUE_HOURS" in workflow assert "/actions/runs?status=${active_status}&per_page=100" in workflow assert "for active_status in queued in_progress" in workflow + assert '"/repos/${head_repo}/git/ref/heads/${encoded_head_ref}"' in workflow + assert "--jq '.object.sha // empty'" in workflow + assert 'open_pr_heads_json="$(\n jq' in workflow + assert "live ref ${head_repo}:${head_ref} could not be resolved safely" in workflow assert '"pull_request" or .event == "pull_request_target"' in workflow assert "$current_pr_head == null or .head_sha != $current_pr_head" in workflow assert ".head_sha != $current_default_sha" in workflow From 7348cf6f5262209df63c5cc76d16597064adb025 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 14:24:33 +0900 Subject: [PATCH 2/3] fix(actions): fail closed on malformed PR heads --- .github/workflows/pr-review-merge-scheduler.yml | 14 ++++++++++---- tests/test_required_workflow_queue_contract.py | 3 +++ 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index 881165240..5d7b2f463 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -1076,14 +1076,20 @@ jobs: --paginate \ | jq -sr ' add[] - | select(.head.repo.full_name != null and .head.ref != null) - | [.head.repo.full_name, .head.ref] + | [ + (if (.head.repo.full_name | type) == "string" then .head.repo.full_name else "" end), + (if (.head.ref | type) == "string" then .head.ref else "" end) + ] | @tsv ' )"; then while IFS=$'\t' read -r head_repo head_ref; do - [ -n "$head_repo" ] && [ -n "$head_ref" ] || continue - encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | @uri')" + if [ -z "$head_repo" ] || [ -z "$head_ref" ]; then + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: an open PR has a malformed head repository or ref. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + break + fi + encoded_head_ref="$(jq -rn --arg value "$head_ref" '$value | split("/") | map(@uri) | join("/")')" if ! live_head_sha="$( gh api \ -H "Accept: application/vnd.github+json" \ diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 67c2fe69b..78b046918 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -749,9 +749,12 @@ def test_org_queue_sweep_covers_target_repositories_on_a_heartbeat() -> None: assert "/actions/runs?status=${active_status}&per_page=100" in workflow assert "for active_status in queued in_progress" in workflow assert '"/repos/${head_repo}/git/ref/heads/${encoded_head_ref}"' in workflow + assert '$value | split("/") | map(@uri) | join("/")' in workflow assert "--jq '.object.sha // empty'" in workflow assert 'open_pr_heads_json="$(\n jq' in workflow assert "live ref ${head_repo}:${head_ref} could not be resolved safely" in workflow + assert "an open PR has a malformed head repository or ref" in workflow + assert "select(.head.repo.full_name != null and .head.ref != null)" not in workflow assert '"pull_request" or .event == "pull_request_target"' in workflow assert "$current_pr_head == null or .head_sha != $current_pr_head" in workflow assert ".head_sha != $current_default_sha" in workflow From 7c69378fcb84073920b56d0a8dcb4a5a67b23b1c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 23:33:57 -0700 Subject: [PATCH 3/3] fix(scheduler): bound live ref lookups --- .../workflows/pr-review-merge-scheduler.yml | 15 +++++++++++- .../test_required_workflow_queue_contract.py | 24 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-review-merge-scheduler.yml b/.github/workflows/pr-review-merge-scheduler.yml index 71befbd4f..538b28172 100644 --- a/.github/workflows/pr-review-merge-scheduler.yml +++ b/.github/workflows/pr-review-merge-scheduler.yml @@ -604,6 +604,11 @@ jobs: # this contract). The scheduler paginates, so 1000 keeps the practical # GitHub queue ceiling while avoiding an arbitrary per-repository sample. ORG_SWEEP_MAX_PRS: ${{ github.event.client_payload.max_prs || inputs.max_prs || vars.ORG_SWEEP_MAX_PRS || '1000' }} + # Resolving live refs protects against stale pull-request payloads, but + # each lookup consumes one REST request. Above this independent bound, + # skip destructive cancellation for the repository instead of exhausting + # the organization sweep's API/runtime budget. + ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS: ${{ vars.ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS || '100' }} ORG_SWEEP_REVIEW_DISPATCH_LIMIT: ${{ github.event.client_payload.review_dispatch_limit || inputs.review_dispatch_limit || vars.ORG_SWEEP_REVIEW_DISPATCH_LIMIT || '1' }} ORG_SWEEP_BRANCH_UPDATE_LIMIT: ${{ github.event.client_payload.branch_update_limit || inputs.branch_update_limit || vars.ORG_SWEEP_BRANCH_UPDATE_LIMIT || '1' }} ORG_SWEEP_TRIGGER_REVIEWS: ${{ github.event_name == 'schedule' || github.event_name == 'repository_dispatch' && github.event.client_payload.trigger_reviews != false || inputs.trigger_reviews == true }} @@ -832,6 +837,10 @@ jobs: echo "::error::ORG_SWEEP_BRANCH_UPDATE_LIMIT must be -1 or a non-negative integer; got '${ORG_SWEEP_BRANCH_UPDATE_LIMIT}'. Fix the ORG_SWEEP_BRANCH_UPDATE_LIMIT repository variable." exit 1 fi + if ! [[ "$ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS" =~ ^[1-9][0-9]*$ ]]; then + echo "::error::ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS must be a positive integer; got '${ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS}'. Fix the ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS repository variable." + exit 1 + fi # Unset in production (see the env-block comment above). Primary # source: a persistent `ORG_SWEEP_ROTATION_COUNTER` repository # variable on this (.github) repository, incremented by exactly @@ -1083,7 +1092,11 @@ jobs: | @tsv ' )"; then - if [ -n "$open_pr_refs_tsv" ]; then + open_pr_ref_count="$(printf '%s\n' "$open_pr_refs_tsv" | awk 'NF { count += 1 } END { print count + 0 }')" + if (( open_pr_ref_count > ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS )); then + echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: ${open_pr_ref_count} open PR refs exceed the live-ref lookup limit ${ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS}. No run will be cancelled from incomplete evidence." + queue_hygiene_ready=false + elif [ -n "$open_pr_refs_tsv" ]; then while IFS=$'\t' read -r head_repo head_ref; do if [ -z "$head_repo" ] || [ -z "$head_ref" ]; then echo "::warning::Current-HEAD cancellation skipped for ${repo_full_name}: an open PR has a malformed head repository or ref. No run will be cancelled from incomplete evidence." diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 31d61513e..c2f8c7bf8 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -889,6 +889,30 @@ def test_org_queue_sweep_empty_pr_ref_list_skips_the_ref_loop() -> None: assert 'done <<<"$open_pr_refs_tsv"\n fi' in ref_loop +def test_org_queue_sweep_bounds_live_ref_lookups_and_fails_closed() -> None: + """Large queues must not turn hygiene into an unbounded ref API fan-out.""" + workflow = workflow_text("pr-review-merge-scheduler.yml") + ref_loop = workflow.split('if open_pr_refs_tsv="$(\n', 1)[1].split( + ' else\n echo "::warning::Current-HEAD cancellation skipped', + 1, + )[0] + + assert ( + "ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS: ${{ " + "vars.ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS || '100' }}" + ) in workflow + assert '"$ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS" =~ ^[1-9][0-9]*$' in workflow + assert 'open_pr_ref_count="$(printf \'%s\\n\' "$open_pr_refs_tsv"' in ref_loop + assert ( + 'if (( open_pr_ref_count > ORG_QUEUE_HYGIENE_MAX_REF_LOOKUPS )); then' + in ref_loop + ) + assert "live-ref lookup limit" in ref_loop + assert ref_loop.index("queue_hygiene_ready=false") < ref_loop.index( + "while IFS=$'\\t' read -r head_repo head_ref; do" + ) + + def _extract_org_sweep_rotation_snippet(workflow: str) -> str: """Return only the rotation-offset bash block, without the surrounding `gh api`/dispatch logic that would require live network credentials."""