From b0982c8004003f1b51a6a49e9d4905a8cc4595e1 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Tue, 29 Sep 2026 13:55:23 +0000 Subject: [PATCH 1/2] ALCF CI: trigger a fresh GitLab pipeline on re-runs A mirror sync does not create a new pipeline for a SHA that GitLab already has, so re-running the GitHub check re-adopted the old pipeline's result, e.g. one whose jobs failed with stuck_pending_no_matching_runners during an Aurora runner outage. On re-runs, create a fresh pipeline via the trigger token unless one is still active for the SHA. --- .github/workflows/alcf.yml | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.github/workflows/alcf.yml b/.github/workflows/alcf.yml index 51f2af07..bc3ac8b3 100644 --- a/.github/workflows/alcf.yml +++ b/.github/workflows/alcf.yml @@ -90,6 +90,54 @@ jobs: curl -sS -X POST -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$API/mirror/pull" \ || echo "::warning::mirror sync request failed; relying on scheduled sync" + # A mirror sync does not create a new pipeline for a SHA GitLab already has, so a + # plain re-run would re-adopt the old (e.g. runner-outage) result. On re-runs, + # create a fresh pipeline through the trigger token, unless one is still active. + # (Retrying the old pipeline is not an option: the retry would be attributed to + # the bot user, which the Jacamar runners refuse; see header comment.) + - name: Trigger a fresh pipeline on re-run + if: github.run_attempt > 1 && github.event_name != 'pull_request_target' + env: + REF: ${{ github.head_ref || github.ref_name }} + run: | + api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } + + ACTIVE=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \ + | jq -r '.[0] | select(.status | IN("created","waiting_for_resource","preparing","pending","running","scheduled","manual")) | .id // empty' 2>/dev/null || true) + if [ -n "$ACTIVE" ]; then + echo "pipeline $ACTIVE for $HEAD_SHA is still active; adopting it" + exit 0 + fi + if [ -z "$GITLAB_TRIGGER_TOKEN" ]; then + echo "::error::ALCF_GITLAB_TRIGGER_TOKEN secret not set; cannot create a fresh pipeline" + exit 1 + fi + # The trigger runs the ref's current tip, so wait for the mirror to carry + # HEAD_SHA there, and refuse if the branch has moved on in the meantime. + ENC_REF=$(jq -rn --arg r "$REF" '$r | @uri') + for i in $(seq 1 40); do # up to 20 min for the mirror sync + TIP=$(api "$API/repository/branches/$ENC_REF" | jq -r '.commit.id // empty' 2>/dev/null || true) + if [ "$TIP" = "$HEAD_SHA" ]; then break; fi + if [ -n "$TIP" ] && api "$API/repository/compare?from=$HEAD_SHA&to=$TIP" \ + | jq -e '.commits | length > 0' >/dev/null 2>&1; then + echo "::error::$REF has moved past $HEAD_SHA on GitLab ($TIP); re-run the newest commit instead" + exit 1 + fi + sleep 30 + done + if [ "$TIP" != "$HEAD_SHA" ]; then + echo "::error::GitLab mirror of $REF is not at $HEAD_SHA (tip: ${TIP:-none})" + exit 1 + fi + RESP=$(curl -sS -X POST "$API/trigger/pipeline" \ + -F "token=${GITLAB_TRIGGER_TOKEN}" -F "ref=$REF") + ID=$(echo "$RESP" | jq -r '.id // empty' 2>/dev/null || true) + if [ -z "$ID" ]; then + echo "::error::failed to trigger GitLab pipeline: $RESP" + exit 1 + fi + echo "triggered fresh pipeline $ID for $HEAD_SHA" + - name: Wait for pipeline and adopt its result run: | # Transient API failures (network, rate limiting) must not fail the check: From af786a2c14169ecf0c734e9e205089f21b68b261 Mon Sep 17 00:00:00 2001 From: Michel Schanen Date: Tue, 29 Sep 2026 15:03:15 +0000 Subject: [PATCH 2/2] ALCF CI: only re-trigger after an unsuccessful pipeline Trigger a fresh pipeline on re-runs only when the newest pipeline for the head SHA failed, was canceled or was skipped. A missing, active or successful pipeline, or a failed lookup, is left to the wait step, so a transient API failure can no longer create a duplicate pipeline. Replace the mirror wait loop with a single branch-tip check: a finished pipeline for the SHA means GitLab already has the commit, and without one the mirror sync creates the pipeline itself. Also verify that the triggered pipeline runs the head SHA, and pass the ref with --form-string so curl does not interpret it. --- .github/workflows/alcf.yml | 48 ++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/.github/workflows/alcf.yml b/.github/workflows/alcf.yml index bc3ac8b3..cc55d8ca 100644 --- a/.github/workflows/alcf.yml +++ b/.github/workflows/alcf.yml @@ -92,9 +92,10 @@ jobs: # A mirror sync does not create a new pipeline for a SHA GitLab already has, so a # plain re-run would re-adopt the old (e.g. runner-outage) result. On re-runs, - # create a fresh pipeline through the trigger token, unless one is still active. - # (Retrying the old pipeline is not an option: the retry would be attributed to - # the bot user, which the Jacamar runners refuse; see header comment.) + # create a fresh pipeline through the trigger token if the newest one for the SHA + # finished unsuccessfully. (Retrying the old pipeline is not an option: the retry + # would be attributed to the bot user, which the Jacamar runners refuse; see + # header comment.) - name: Trigger a fresh pipeline on re-run if: github.run_attempt > 1 && github.event_name != 'pull_request_target' env: @@ -102,41 +103,32 @@ jobs: run: | api() { curl -sS --max-time 30 -H "PRIVATE-TOKEN: ${GITLAB_TOKEN}" "$@" || true; } - ACTIVE=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \ - | jq -r '.[0] | select(.status | IN("created","waiting_for_resource","preparing","pending","running","scheduled","manual")) | .id // empty' 2>/dev/null || true) - if [ -n "$ACTIVE" ]; then - echo "pipeline $ACTIVE for $HEAD_SHA is still active; adopting it" - exit 0 - fi + # Only replace a pipeline that finished unsuccessfully. Anything else (none + # yet, still active, succeeded, lookup failed) is left to the wait step below. + STATUS=$(api "$API/pipelines?sha=$HEAD_SHA&order_by=id&sort=desc&per_page=1" \ + | jq -r '.[0].status // empty' 2>/dev/null || true) + case "$STATUS" in + failed|canceled|skipped) ;; + *) echo "latest pipeline for $HEAD_SHA: ${STATUS:-none}; nothing to re-trigger"; exit 0 ;; + esac if [ -z "$GITLAB_TRIGGER_TOKEN" ]; then echo "::error::ALCF_GITLAB_TRIGGER_TOKEN secret not set; cannot create a fresh pipeline" exit 1 fi - # The trigger runs the ref's current tip, so wait for the mirror to carry - # HEAD_SHA there, and refuse if the branch has moved on in the meantime. - ENC_REF=$(jq -rn --arg r "$REF" '$r | @uri') - for i in $(seq 1 40); do # up to 20 min for the mirror sync - TIP=$(api "$API/repository/branches/$ENC_REF" | jq -r '.commit.id // empty' 2>/dev/null || true) - if [ "$TIP" = "$HEAD_SHA" ]; then break; fi - if [ -n "$TIP" ] && api "$API/repository/compare?from=$HEAD_SHA&to=$TIP" \ - | jq -e '.commits | length > 0' >/dev/null 2>&1; then - echo "::error::$REF has moved past $HEAD_SHA on GitLab ($TIP); re-run the newest commit instead" - exit 1 - fi - sleep 30 - done + # The trigger runs the ref's current tip, which must still be HEAD_SHA. + TIP=$(api "$API/repository/branches/$(jq -rn --arg r "$REF" '$r | @uri')" \ + | jq -r '.commit.id // empty' 2>/dev/null || true) if [ "$TIP" != "$HEAD_SHA" ]; then - echo "::error::GitLab mirror of $REF is not at $HEAD_SHA (tip: ${TIP:-none})" + echo "::error::$REF is at ${TIP:-unknown} on GitLab, not $HEAD_SHA; re-run the newest commit instead" exit 1 fi RESP=$(curl -sS -X POST "$API/trigger/pipeline" \ - -F "token=${GITLAB_TRIGGER_TOKEN}" -F "ref=$REF") - ID=$(echo "$RESP" | jq -r '.id // empty' 2>/dev/null || true) - if [ -z "$ID" ]; then - echo "::error::failed to trigger GitLab pipeline: $RESP" + --form-string "token=${GITLAB_TRIGGER_TOKEN}" --form-string "ref=$REF" || true) + if [ "$(echo "$RESP" | jq -r '.sha // empty' 2>/dev/null)" != "$HEAD_SHA" ]; then + echo "::error::failed to trigger a GitLab pipeline for $HEAD_SHA: $RESP" exit 1 fi - echo "triggered fresh pipeline $ID for $HEAD_SHA" + echo "triggered fresh pipeline $(echo "$RESP" | jq -r .id) for $HEAD_SHA" - name: Wait for pipeline and adopt its result run: |