diff --git a/.github/actions/pantheon-push/action.yml b/.github/actions/pantheon-push/action.yml index 0a91e32..5de6435 100644 --- a/.github/actions/pantheon-push/action.yml +++ b/.github/actions/pantheon-push/action.yml @@ -106,6 +106,42 @@ runs: echo "Nothing staged after composer install; skipping pre-commit (upstream push will no-op the commit too)." fi + # Pre-flight: ensure the target Pantheon env is in git mode. If it's been + # flipped to SFTP (e.g. someone opened the Dashboard Code view and toggled + # it), the push would be rejected by Pantheon's pre-receive hook — + # `pantheon-systems/push-to-pantheon@0.9.0` swallows that exit code and + # reports the step as successful. We pre-empt the most common cause here; + # the verify-after-push step below catches anything else. + # + # Only applies if the env already exists. New multidevs are created in git + # mode by terminus multidev:create (handled inside the upstream action). + - name: Ensure Pantheon env is in git mode (pre-flight) + shell: bash + env: + TERMINUS_MACHINE_TOKEN: ${{ inputs.machine_token }} + SITE: ${{ inputs.site }} + TARGET_ENV: ${{ inputs.target_env }} + run: | + set -euo pipefail + # Use env:list (an authoritative listing) rather than env:info >/dev/null, + # which would silently swallow auth/network errors as "env doesn't exist". + # If terminus itself errors, set -e fails the step with the real cause. + # Matches the existing pattern in reusable-pantheon-deploy-rc-multidev.yml + # for the live-env probe. + if ! terminus env:list "${SITE}" --format=list | grep -q "^${TARGET_ENV}$"; then + echo "Env ${SITE}.${TARGET_ENV} does not exist yet; multidev:create will provision it in git mode. Skipping connection-mode check." + exit 0 + fi + # No 2>/dev/null fallback — if connection:info errors here (auth, network), + # we want the step to fail with terminus's real error rather than coerce + # the result to an empty string and attempt connection:set on bad data. + MODE=$(terminus connection:info "${SITE}.${TARGET_ENV}" --field=connection_mode) + echo "Current connection mode for ${SITE}.${TARGET_ENV}: ${MODE}" + if [[ "${MODE}" != "git" ]]; then + echo "::warning::${SITE}.${TARGET_ENV} is in '${MODE}' mode. Switching to git so the push isn't rejected by the pre-receive hook." + terminus connection:set "${SITE}.${TARGET_ENV}" git + fi + - name: Push to Pantheon uses: pantheon-systems/push-to-pantheon@0.9.0 env: @@ -127,3 +163,62 @@ runs: # multidev:create, so re-pushes to an existing rc-/epr-/pr- multidev # update the code instead of failing with "environment already exists". skip_build_tools: true + + # Verify-after-push: pantheon-systems/push-to-pantheon@0.9.0 does NOT + # propagate the exit code from `git push`. When Pantheon rejects the push + # (SFTP mode, branch protection, lost-update race, etc.) the action still + # reports the step as success and bobheadxi/deployments@v1 (also invoked + # inside the upstream) marks the GitHub deployment "success". We compare + # the local HEAD we meant to push against the remote ref Pantheon now has, + # and fail the workflow loudly if they don't match. Belt-and-braces: + # catches the SFTP case our pre-flight already handles AND any future + # silent-fail mode in the upstream action. + # + # Requires the SSH agent loaded by the upstream's webfactory/ssh-agent + # step (SSH_AUTH_SOCK is in GITHUB_ENV, so it persists into this step). + - name: Verify push landed on Pantheon + shell: bash + env: + TERMINUS_MACHINE_TOKEN: ${{ inputs.machine_token }} + SITE: ${{ inputs.site }} + TARGET_ENV: ${{ inputs.target_env }} + run: | + set -euo pipefail + LOCAL_HEAD=$(git rev-parse HEAD) + PANTHEON_GIT_URL=$(terminus connection:info "${SITE}.${TARGET_ENV}" --field=git_url) + if [[ -z "${PANTHEON_GIT_URL}" ]]; then + echo "::error::Could not determine Pantheon git URL for ${SITE}.${TARGET_ENV}; cannot verify push." + exit 1 + fi + echo "Local HEAD: ${LOCAL_HEAD}" + echo "Pantheon URL: ${PANTHEON_GIT_URL}" + # Retry ls-remote a few times: for freshly created multidevs there can + # be a short window where Pantheon has accepted the push but the ref + # isn't yet visible via ls-remote (related to the propagation race the + # post-deploy drush step also defends against). Three attempts with + # short backoff covers the typical lag without papering over a real + # silent-fail. + REMOTE_HEAD="" + for attempt in 1 2 3; do + REMOTE_HEAD=$(git ls-remote "${PANTHEON_GIT_URL}" "refs/heads/${TARGET_ENV}" | awk '{print $1}') + if [[ -n "${REMOTE_HEAD}" ]]; then + break + fi + if [[ ${attempt} -lt 3 ]]; then + SLEEP=$((attempt * 5)) + echo "ls-remote returned no ref on attempt ${attempt}; retrying in ${SLEEP}s (Pantheon ref propagation lag)..." + sleep "${SLEEP}" + fi + done + echo "Pantheon HEAD: ${REMOTE_HEAD:-}" + if [[ -z "${REMOTE_HEAD}" ]]; then + echo "::error::Pantheon ${TARGET_ENV} has no commits on refs/heads/${TARGET_ENV} after 3 attempts. The upstream push silently failed or the branch was never created." + echo "::error::Check the previous step's output for 'remote rejected' / 'pre-receive hook declined' / SFTP-mode notices." + exit 1 + fi + if [[ "${LOCAL_HEAD}" != "${REMOTE_HEAD}" ]]; then + echo "::error::Pantheon ${TARGET_ENV} is at ${REMOTE_HEAD}, expected ${LOCAL_HEAD}. The upstream push silently failed." + echo "::error::Check the previous step's output for 'remote rejected' / 'pre-receive hook declined' / SFTP-mode notices." + exit 1 + fi + echo "✅ Pantheon ${TARGET_ENV} is at ${LOCAL_HEAD} as expected." diff --git a/.github/workflows/reusable-pantheon-deploy-dev.yml b/.github/workflows/reusable-pantheon-deploy-dev.yml index 42e1988..b986ab4 100644 --- a/.github/workflows/reusable-pantheon-deploy-dev.yml +++ b/.github/workflows/reusable-pantheon-deploy-dev.yml @@ -62,7 +62,7 @@ jobs: semantic_release: name: Semantic Release if: ${{ inputs.run_semantic_release }} - uses: Square360/shared-workflows/.github/workflows/reusable-semantic-release.yml@v3.2.3 + uses: Square360/shared-workflows/.github/workflows/reusable-semantic-release.yml@v3.2.4 secrets: CI_GH_TOKEN: ${{ secrets.CI_GH_TOKEN }} @@ -92,7 +92,7 @@ jobs: fetch-depth: 1 - name: Install Terminus - uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.3 + uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.4 with: machine_token: ${{ secrets.PANTHEON_MACHINE_TOKEN }} @@ -102,7 +102,7 @@ jobs: # ----------------------------------------------------------------------- - name: Push code to Pantheon DEV id: deploy_pantheon - uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.4 with: site: ${{ inputs.pantheon_site }} target_env: dev @@ -159,7 +159,7 @@ jobs: - name: Run post-deploy drush commands id: drush_commands - uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.4 with: site: ${{ inputs.pantheon_site }} env: dev diff --git a/.github/workflows/reusable-pantheon-deploy-epic-multidev.yml b/.github/workflows/reusable-pantheon-deploy-epic-multidev.yml index c27efd9..490ce9f 100644 --- a/.github/workflows/reusable-pantheon-deploy-epic-multidev.yml +++ b/.github/workflows/reusable-pantheon-deploy-epic-multidev.yml @@ -126,7 +126,7 @@ jobs: echo "Epic multidev: ${TARGET_ENV}" - name: Install Terminus - uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.3 + uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.4 with: machine_token: ${{ secrets.PANTHEON_MACHINE_TOKEN }} @@ -148,7 +148,7 @@ jobs: - name: Push code to Pantheon epic multidev id: deploy - uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.4 with: site: ${{ inputs.pantheon_site }} target_env: ${{ steps.derive.outputs.target_env }} @@ -159,7 +159,7 @@ jobs: - name: Run post-deploy drush commands id: drush_commands - uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.4 with: site: ${{ inputs.pantheon_site }} env: ${{ steps.derive.outputs.target_env }} @@ -234,7 +234,7 @@ jobs: # The security workflow detects epr-* internally and runs unconditionally # in strict mode (fails on any High-severity finding) — same gate as rc-*, # per the Square360 SOP for epic integration multidevs. No force_run needed. - uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-security-scan.yml@v3.2.3 + uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-security-scan.yml@v3.2.4 with: pantheon_site: ${{ inputs.pantheon_site }} target_env: ${{ needs.deploy_epic_multidev.outputs.target_env }} diff --git a/.github/workflows/reusable-pantheon-deploy-pr-multidev.yml b/.github/workflows/reusable-pantheon-deploy-pr-multidev.yml index 5c138f0..16ef227 100644 --- a/.github/workflows/reusable-pantheon-deploy-pr-multidev.yml +++ b/.github/workflows/reusable-pantheon-deploy-pr-multidev.yml @@ -83,7 +83,7 @@ jobs: run: echo "target_env=pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT - name: Install Terminus - uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.3 + uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.4 with: machine_token: ${{ secrets.PANTHEON_MACHINE_TOKEN }} @@ -105,7 +105,7 @@ jobs: - name: Push code to Pantheon multidev id: deploy - uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.4 with: site: ${{ inputs.pantheon_site }} target_env: ${{ steps.derive.outputs.target_env }} @@ -116,7 +116,7 @@ jobs: - name: Run post-deploy drush commands id: drush_commands - uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.4 with: site: ${{ inputs.pantheon_site }} env: ${{ steps.derive.outputs.target_env }} @@ -176,7 +176,11 @@ jobs: fi - name: Comment on PR with multidev URL - if: ${{ github.event_name == 'pull_request' }} + # Gate on success() so the "🚀 Multidev Environment Ready!" comment + # doesn't get posted when the deploy actually failed — including the + # case where pantheon-push's verify-after-push step caught the + # upstream action's silent-fail mode (SFTP-mode rejection, etc.). + if: ${{ success() && github.event_name == 'pull_request' }} uses: actions/github-script@v8 with: script: | @@ -224,7 +228,7 @@ jobs: contains(github.event.pull_request.body, '[run-security]') || contains(github.event.pull_request.body, '[security]') }} - uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-security-scan.yml@v3.2.3 + uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-security-scan.yml@v3.2.4 with: pantheon_site: ${{ inputs.pantheon_site }} target_env: ${{ needs.deploy_pr_multidev.outputs.target_env }} @@ -252,7 +256,7 @@ jobs: contains(github.event.pull_request.body, '[run-vrt]') || contains(github.event.pull_request.body, '[vrt]') }} - uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-vrt.yml@v3.2.3 + uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-vrt.yml@v3.2.4 with: pantheon_site: ${{ inputs.pantheon_site }} target_env: ${{ needs.deploy_pr_multidev.outputs.target_env }} diff --git a/.github/workflows/reusable-pantheon-deploy-rc-multidev.yml b/.github/workflows/reusable-pantheon-deploy-rc-multidev.yml index 697e2d1..43c83e9 100644 --- a/.github/workflows/reusable-pantheon-deploy-rc-multidev.yml +++ b/.github/workflows/reusable-pantheon-deploy-rc-multidev.yml @@ -88,7 +88,7 @@ jobs: echo "Derived RC multidev name: ${TARGET_ENV}" - name: Install Terminus - uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.3 + uses: Square360/shared-workflows/.github/actions/terminus-install@v3.2.4 with: machine_token: ${{ secrets.PANTHEON_MACHINE_TOKEN }} @@ -111,7 +111,7 @@ jobs: - name: Push code to Pantheon multidev id: deploy - uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-push@v3.2.4 with: site: ${{ inputs.pantheon_site }} target_env: ${{ steps.derive.outputs.target_env }} @@ -122,7 +122,7 @@ jobs: - name: Run post-deploy drush commands id: drush_commands - uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.3 + uses: Square360/shared-workflows/.github/actions/pantheon-post-deploy-drush@v3.2.4 with: site: ${{ inputs.pantheon_site }} env: ${{ steps.derive.outputs.target_env }} @@ -165,7 +165,7 @@ jobs: needs: deploy_rc_multidev # The security workflow detects rc-* internally and runs unconditionally # in strict mode (fails on any High-severity finding). No force_run needed. - uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-security-scan.yml@v3.2.3 + uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-security-scan.yml@v3.2.4 with: pantheon_site: ${{ inputs.pantheon_site }} target_env: ${{ needs.deploy_rc_multidev.outputs.target_env }} @@ -193,7 +193,7 @@ jobs: contains(github.event.head_commit.message, '[run-vrt]') || contains(github.event.head_commit.message, '[vrt]') }} - uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-vrt.yml@v3.2.3 + uses: Square360/shared-workflows/.github/workflows/reusable-pantheon-vrt.yml@v3.2.4 with: pantheon_site: ${{ inputs.pantheon_site }} target_env: ${{ needs.deploy_rc_multidev.outputs.target_env }} diff --git a/docs/CHANGELOG-AGENT.md b/docs/CHANGELOG-AGENT.md index 0640701..2dd0fd5 100644 --- a/docs/CHANGELOG-AGENT.md +++ b/docs/CHANGELOG-AGENT.md @@ -2,6 +2,36 @@ This file tracks the work done together with GitHub Copilot on the Square360 Shared Pantheon Workflows repository. +## 2026-05-22 + +### Fix - `pantheon-push` fails loudly when Pantheon rejects the push (v3.2.4) + +- **The problem (YH-571 / Yale-Health-Drupal PR #126):** `pantheon-systems/push-to-pantheon@0.9.0` does not propagate the exit code from `git push`. When Pantheon's pre-receive hook rejects the push (env in SFTP mode, branch protection, lost-update race, etc.), the upstream action reports the step as successful, `bobheadxi/deployments@v1` (invoked inside the upstream) marks the GitHub deployment "success," and the consumer workflow continues — posting "🚀 Multidev Environment Ready!" on the PR while Pantheon still has the old SHA. Discovered when a security fix appeared deployed but `drush deploy` ran on stale code and re-installed a module we'd intentionally removed. +- **Two defensive layers added to `.github/actions/pantheon-push/action.yml`:** + 1. **Pre-flight SFTP-mode flip** — before invoking the upstream push, check `terminus connection:info . --field=connection_mode` and call `terminus connection:set . git` if it's anything other than `git`. Catches the most common cause of the silent rejection. Skipped when the env doesn't exist yet (multidev:create provisions in git mode). + 2. **Verify-after-push** — after the upstream push step, compare `git rev-parse HEAD` (our local artifact commit) against `git ls-remote refs/heads/`. If they differ, fail the workflow with a clear actionable error pointing the developer at the previous step's rejection notice. Belt-and-braces: catches *any* future silent-fail mode in the upstream, not just SFTP. +- **`if: success()` gate on the PR-comment step** in `.github/workflows/reusable-pantheon-deploy-pr-multidev.yml` so the "Multidev Environment Ready!" PR comment doesn't post when the deploy actually failed. +- **What this doesn't fix:** the `bobheadxi/deployments@v1` "deployment status: success" call lives inside the upstream `pantheon-systems/push-to-pantheon@0.9.0` action and runs before our verify step. The deployment record on the GitHub Deployments tab will still show success briefly until the workflow itself goes red. Fixing that requires either forking the upstream or adding a follow-up `bobheadxi/deployments@v1` "set status to failure" step in our reusable workflows — deferred. The job-level red state is enough to break the false-positive in CI. +- **Out-of-scope upstream issue:** the underlying bug should also be filed upstream at `pantheon-systems/push-to-pantheon`, but that won't unblock us on any reasonable timeframe. +- See `docs/todo/pantheon-push-swallows-rejection.md` (marked RESOLVED in v3.2.4) for the full investigation. + +### Maintenance - Self-reference bumps to v3.2.4 + +- **Bumped all in-repo `uses:` self-references from `@v3.2.3` to `@v3.2.4`** across: + - `reusable-pantheon-deploy-dev.yml` + - `reusable-pantheon-deploy-pr-multidev.yml` + - `reusable-pantheon-deploy-rc-multidev.yml` + - `reusable-pantheon-deploy-epic-multidev.yml` +- Affects: `terminus-install`, `pantheon-push`, `pantheon-post-deploy-drush`, `reusable-semantic-release.yml`, `reusable-pantheon-security-scan.yml`, `reusable-pantheon-vrt.yml` +- IDE flags these as "Unable to resolve" until the v3.2.4 tag exists — expected; semantic-release tags on merge to `main` and the diagnostics resolve themselves. + +### Validation plan (manual, in Yale Health) + +1. Merge to `main` → semantic-release tags `v3.2.4` +2. **Don't yet** bump `pantheon-github-workflows` template SHAs or push the consumer composer plugin — point Yale Health workflows manually at `@v3.2.4` first +3. Reproduce the original failure mode: `terminus connection:set yalehealth-yale-edu.pr-126 sftp`, re-trigger the workflow, confirm it now fails loudly with the pre-flight warning OR the verify-after-push error (the pre-flight should catch SFTP first; the verify-after-push is the catch-all) +4. Once verified in Yale Health, do the full release chain: capture SHA → bump `pantheon-github-workflows` templates → Satis rebuild → composer plugin version bump + ## 2026-05-21 ### Feature - `pantheon-post-deploy-drush` switches to `drush deploy` (v3.2.0) diff --git a/docs/todo/pantheon-push-swallows-rejection.md b/docs/todo/pantheon-push-swallows-rejection.md new file mode 100644 index 0000000..668eb2e --- /dev/null +++ b/docs/todo/pantheon-push-swallows-rejection.md @@ -0,0 +1,195 @@ +# `pantheon-push` action silently succeeds when Pantheon rejects the push + +**Status:** ✅ RESOLVED in v3.2.4 (2026-05-22) — implementation took Options A + B from the "Suggested fixes" section below: pre-flight SFTP-mode flip in `.github/actions/pantheon-push/action.yml`, plus a verify-after-push step that compares local HEAD to `git ls-remote` and fails the workflow loudly on mismatch. PR-comment step in `reusable-pantheon-deploy-pr-multidev.yml` also gated with `if: success()`. The bobheadxi deployment-status note in this doc was slightly off — that step lives inside the upstream action, not in our reusable workflows, so gating it would require forking upstream (deferred; not blocking the false-positive fix). +**Originally discovered:** 2026-05-22 (YH-571 session, yalehealth-yale-edu PR #126) +**Originally affected:** `Square360/shared-workflows@v3.2.2` (and all earlier 3.x — the upstream bug remains; our wrapper now defends against it) +**Blast radius:** every Square360 site that uses `pantheon-push` / `reusable-pantheon-deploy-pr-multidev.yml` / `reusable-pantheon-deploy-rc-multidev.yml` / `reusable-pantheon-deploy-epic-multidev.yml` + +--- + +## TL;DR + +When the Pantheon git remote rejects a push (pre-receive hook declined — e.g. environment in SFTP mode, branch protection on master/live, locked artifact tag, lost-update race), our `pantheon-push` composite action **reports the workflow step as successful**, the deployment record is set to `status: success`, the PR gets a green "Deploy to PR Multidev / Deploy to PR Multidev pass" check, and a "🚀 Multidev Environment Ready!" comment is posted. Pantheon never received the code. + +Every downstream consequence of this is invisible until someone manually verifies. In the discovery case, `drush deploy` ran on stale code, `drush cim` re-installed a module we'd intentionally removed (because the multidev's on-disk `config/sync/core.extension.yml` was the pre-fix version), and a security finding stayed open in production for hours longer than it should have. + +**This is a false-positive in CI for code deploys. It must fail loudly.** + +--- + +## Evidence + +### Workflow run that exposed the bug + +Site: `yalehealth-yale-edu` +PR: +Run: +Job: `Deploy to PR Multidev / Deploy to PR Multidev` (job id 77429080975) +Logs (saved locally for diagnosis): `_working/github_logs/logs_70362116371/` + +### The exact log lines (from the push step) + +File: `_working/github_logs/logs_70362116371/Deploy to PR Multidev _ Deploy to PR Multidev/6_Push code to Pantheon multidev.txt` + +Lines 783-793 (the rejection): + +``` +2026-05-22T17:20:30.3701 remote: Environment 'pr-126' (branch: pr-126) is currently in SFTP mode. +2026-05-22T17:20:30.3702 +2026-05-22T17:20:30.3703 remote: If you are trying to push changes to a different branch or environment, try: +2026-05-22T17:20:30.3704 remote: git push origin [branch-name] +2026-05-22T17:20:30.3705 remote: Switch connection mode to Git via the Pantheon Dashboard or Terminus: +2026-05-22T17:20:30.3707 remote: terminus connection:set .pr-126 git +2026-05-22T17:20:30.4131 To ssh://codeserver.dev.../repository.git +2026-05-22T17:20:30.4133 ! [remote rejected] HEAD -> pr-126 (pre-receive hook declined) +2026-05-22T17:20:30.4134 error: failed to push some refs to 'ssh://codeserver.dev.../repository.git' +``` + +Immediately followed by (no intervening error handling): + +``` +2026-05-22T17:20:30.4191 ##[group]Run bobheadxi/deployments@v1 +2026-05-22T17:20:30.4192 with: +2026-05-22T17:20:30.4193 status: success <-- hardcoded literal +2026-05-22T17:20:30.4194 env: pr-126 +2026-05-22T17:20:30.4195 env_url: https://pr-126-yalehealth-yale-edu.pantheonsite.io +``` + +And then: + +``` +2026-05-22T17:20:32.9112 finishing deployment for 4786335224 with status success +2026-05-22T17:20:33.2651 4786335224 status set to success { statusID: 13650072219 } +``` + +### Independent verification + +After the workflow "succeeded," we confirmed Pantheon had the OLD commit: + +- GitHub's `YH-571` HEAD: `8a6a5837` (the latest, with the security fixes) +- Pantheon's `pr-126` ref: `d0212ed6` (three commits behind, no fixes deployed) + +Confirmed via `git ls-remote ssh://codeserver.dev.../repository.git refs/heads/pr-126`. + +Confirmed via deployed `config/sync/core.extension.yml` on the container — still listing `jsonapi: 0` (the entry we'd removed in the latest commit). + +--- + +## Where the bug actually lives + +The Square360 composite action [.github/actions/pantheon-push/action.yml](../../.github/actions/pantheon-push/action.yml) delegates the actual push to the upstream third-party action: + +```yaml +- name: Push to Pantheon + uses: pantheon-systems/push-to-pantheon@0.9.0 +``` + +That upstream action's internal script `${GITHUB_ACTION_PATH}/scripts/main.sh push_to_pantheon` is what runs the git push and prints the rejection. It is **not** propagating the git push's non-zero exit code to the calling step. + +Suspected cause inside `pantheon-systems/push-to-pantheon@0.9.0`: a `|| true`, an unchecked `$?`, or a `git push 2>&1 | tee ...` pipeline where `pipefail` is not set so the pipeline exits with the success status of `tee`. Without access to the action's source pinned at that exact SHA, the precise line is conjecture — but the symptom is unambiguous. + +Square360 owns the wrapper; we don't own the upstream. So the fix has to be defensive on our side. + +--- + +## Why the workflow has shown green for everyone, every time, until now + +Two factors made this latent rather than visible: + +1. **The push usually succeeds.** Most multidevs land in git mode (Pantheon's default for new envs and for envs created by the Build Tools plugin). The pre-receive hook only kicks in on SFTP-mode envs, on master/live (when the env requires a deploy not a push), and on certain tag/branch protection conditions. +2. **`pantheon-push` doesn't currently run any post-push verification.** There's no "did Pantheon actually advance to my SHA" check. So when the silent failure mode triggers, no other step in the pipeline catches it. + +In the discovery case, pr-126 was inadvertently flipped to SFTP mode (likely by a contributor opening the Pantheon Dashboard's Code view, which can quietly toggle mode). Every push for hours after that silently failed; CI kept reporting green. + +--- + +## Suggested fixes (in increasing order of effort and confidence) + +### Option A — Verify-after-push (recommended, low risk) + +After the `pantheon-systems/push-to-pantheon@0.9.0` step, add a verification step in our composite that: + +1. Reads the local HEAD SHA we *meant* to push +2. Reads the remote ref Pantheon now has via `git ls-remote` against the Pantheon SSH URL (the SSH agent is still loaded from the push step) +3. Fails the workflow with a clear error if they don't match + +Rough shape: + +```yaml +- name: Verify push landed on Pantheon + shell: bash + env: + SITE: ${{ inputs.site }} + TARGET_ENV: ${{ inputs.target_env }} + run: | + set -euo pipefail + LOCAL_HEAD=$(git rev-parse HEAD) + PANTHEON_GIT_URL=$(terminus connection:info "${SITE}.${TARGET_ENV}" --field=git_url) + REMOTE_HEAD=$(git ls-remote "${PANTHEON_GIT_URL}" "refs/heads/${TARGET_ENV}" | awk '{print $1}') + echo "Local HEAD: ${LOCAL_HEAD}" + echo "Pantheon HEAD: ${REMOTE_HEAD}" + if [[ "${LOCAL_HEAD}" != "${REMOTE_HEAD}" ]]; then + echo "::error::Pantheon ${TARGET_ENV} is at ${REMOTE_HEAD}, expected ${LOCAL_HEAD}. The upstream push silently failed." + echo "::error::Check the previous step's output for 'remote rejected' / 'pre-receive hook declined' / SFTP-mode notices." + exit 1 + fi + echo "✅ Pantheon ${TARGET_ENV} is at ${LOCAL_HEAD} as expected." +``` + +This catches the symptom without needing to debug or fork the upstream action. It also catches *any* future silent-fail mode in `pantheon-systems/push-to-pantheon`, not just this specific one. Belt-and-braces. + +Edge cases to handle: +- The push deliberately advances Pantheon past the local HEAD (because upstream commits an artifact). Account for that: the post-stage commit we make in our action becomes the local HEAD, so the comparison is `git rev-parse HEAD` *after* the artifact commit, not the original GitHub SHA. The action.yml already commits before invoking push-to-pantheon, so `git rev-parse HEAD` at verify time is the right value. +- Multidev creation flow: when a new multidev is being created, the ref may not exist on Pantheon yet at the moment we read it. Handle the `ls-remote` returning empty as "creation in progress, accept" or wait+retry. + +### Option B — Detect SFTP mode before push (preventative) + +Before invoking push-to-pantheon, run: + +```bash +MODE=$(terminus connection:info "${SITE}.${TARGET_ENV}" --field=connection_mode) +if [[ "${MODE}" != "git" ]]; then + echo "::warning::${SITE}.${TARGET_ENV} is in ${MODE} mode. Flipping to git." + terminus connection:set "${SITE}.${TARGET_ENV}" git +fi +``` + +Catches one specific cause of the rejection. Doesn't help with branch protection, lost-update races, or any other rejection mode. Combine with Option A. + +### Option C — Patch / fork the upstream action + +Replace `pantheon-systems/push-to-pantheon@0.9.0` with a Square360-owned variant (`Square360/push-to-pantheon` or vendored under `.github/actions/`) that propagates exit codes correctly. Highest effort, highest control. Worth it if the upstream is otherwise unmaintained, but Options A+B together likely give the same defensive guarantees at lower cost. + +### Option D — Upstream fix (parallel track, no blocking) + +File an issue at describing the swallowed exit code with the evidence above. Won't unblock Square360 in any reasonable timeframe but should land eventually. + +--- + +## Acceptance criteria for this fix + +1. A push that Pantheon rejects (any reason) **fails the GitHub Actions job** loudly. The "Deploy to PR Multidev" check goes red. No PR gets a green checkmark for a deploy that didn't actually deploy. +2. The error message names the actual cause when detectable (SFTP mode, branch protection, etc.) or directs the developer to the previous step's log otherwise. +3. The "🚀 Multidev Environment Ready!" PR comment is not posted on a failed deploy. (The current workflow may need an `if: success()` guard added on the comment step too.) +4. The GitHub deployment record (`bobheadxi/deployments@v1`) does not get set to `status: success` on a failed push — `if: success()` on that step too, or rely on `set -e` propagating from the new verify step. +5. Regression test: temporarily put a Pantheon multidev in SFTP mode, trigger the workflow, confirm the run fails with a clear actionable error. + +--- + +## Related context for whoever picks this up + +- **Affected workflow files in this repo**: `.github/workflows/reusable-pantheon-deploy-pr-multidev.yml`, `.github/workflows/reusable-pantheon-deploy-rc-multidev.yml`, `.github/workflows/reusable-pantheon-deploy-epic-multidev.yml` all use the `pantheon-push` composite. +- **The composite action**: [.github/actions/pantheon-push/action.yml](../../.github/actions/pantheon-push/action.yml). The "Push to Pantheon" step at the bottom is where the verify-after-push step should be appended. +- **Where the "Multidev Environment Ready" comment is posted**: in the reusable workflow files above (after the `pantheon-push` step). Each one has an `actions/github-script@v8` step that posts the PR comment. Add `if: success()` to that step too. +- **Where the deployment status is set to success**: `bobheadxi/deployments@v1` step, also in the reusable workflow files. The `with.status: success` is literal; the step itself runs unconditionally because there's no `if: success()` gate. +- **The yalehealth-yale-edu side workaround we're applying right now**: `terminus connection:set yalehealth-yale-edu.pr-126 git` then re-run the workflow. This unblocks PR #126 specifically but doesn't fix the workflow bug. +- **George's instruction (2026-05-22)**: GitHub is the source of record for Square360 projects; never propose direct-pushing to Pantheon as a workflow workaround. This bug, when fixed, removes one of the few legitimate reasons someone might be tempted to bypass GitHub. + +--- + +## File this work somewhere visible + +Once the fix lands, update: +- `docs/CHANGELOG-AGENT.md` and `docs/CHANGELOG.md` with the fix description and which version it landed in +- `docs/TODO.md` if it has a "known issues" section — remove the entry +- Drop a heads-up in `#client_*` Slack channels or whatever the rollout-comms surface is, so site maintainers know to expect newly-red CI on multidev/dev/test/live envs that were quietly broken until now