From 7491bd8d5e909e364d2946cd2617ac864df4d917 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 21 Sep 2026 22:21:06 +0000 Subject: [PATCH 1/3] chore(main): release 0.41.0 --- .release-please-manifest.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 20c888bd6..dbe1b2bfb 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.40.8" + ".": "0.41.0" } From a3c14528a3822cb4e86aacaf3fa97f2f81dd585f Mon Sep 17 00:00:00 2001 From: Wikid82 Date: Mon, 21 Sep 2026 17:59:20 -0400 Subject: [PATCH 2/3] chore: fix nightly health check false negatives and dispatch deadlock The weekly nightly->main promotion's health check only looked at completed workflow runs and gave up after ~2 minutes of polling, so an E2E run still in progress (real runtime ~20min) was indistinguishable from one that never started, producing false-negative "unhealthy" verdicts and blocking promotion. Separately, trigger-required-checks existed to dispatch missing required workflows but was gated behind create-promotion-pr, which only runs when the branch is already healthy - so it could never fire in exactly the case where a missing E2E/CodeQL run was the problem. Rewrite the health check to poll up to 40 minutes with no status filter (so in-progress runs are visible), and to self-dispatch missing E2E/CodeQL runs on the nightly HEAD directly instead of depending on the deadlocked downstream job. Quality Checks has no workflow_dispatch trigger, so it is never dispatched - if still missing at the deadline that's treated as a real failure, not a timing artifact. Trim the now-redundant E2E/CodeQL entries from trigger-required-checks. --- .../workflows/weekly-nightly-promotion.yml | 210 +++++++++++------- 1 file changed, 133 insertions(+), 77 deletions(-) diff --git a/.github/workflows/weekly-nightly-promotion.yml b/.github/workflows/weekly-nightly-promotion.yml index 2eb6f9175..9ea54d03a 100644 --- a/.github/workflows/weekly-nightly-promotion.yml +++ b/.github/workflows/weekly-nightly-promotion.yml @@ -39,6 +39,10 @@ jobs: check-nightly-health: name: Verify Nightly Branch Health runs-on: ubuntu-latest + timeout-minutes: 50 + permissions: + actions: write + contents: read outputs: is_healthy: ${{ steps.check.outputs.is_healthy }} latest_run_url: ${{ steps.check.outputs.latest_run_url }} @@ -76,114 +80,164 @@ jobs: // Check critical workflows on the current nightly HEAD only. // Nightly build itself is scheduler-driven and not a reliable per-commit gate. + // + // `dispatchable: false` means the workflow only triggers on push/pull_request + // (no workflow_dispatch trigger) - we must never attempt to dispatch it + // (that would 422), we can only wait and, if it's still missing at the + // deadline, treat that as a real failure rather than a timing artifact. const criticalWorkflows = [ { workflowFile: 'quality-checks.yml', fallbackNames: ['Quality Checks'], + dispatchable: false, }, { workflowFile: 'e2e-tests-split.yml', fallbackNames: ['E2E Tests'], + dispatchable: true, }, { workflowFile: 'codeql.yml', fallbackNames: ['CodeQL - Analyze'], + dispatchable: true, }, ]; - // Retry window to avoid race conditions where required checks are not yet materialized. - const maxAttempts = 6; - const waitMs = 20000; + // Poll for up to 40 minutes - comfortably above E2E's observed ~20 minute + // real runtime - so an in-progress run isn't mistaken for "never triggered". + // Missing dispatchable workflows are self-dispatched on nightly HEAD here, + // rather than relying on trigger-required-checks downstream, which can never + // run in exactly the case where that's needed (it's gated behind is_healthy). + const pollIntervalMs = 60_000; + const maxWaitMs = 40 * 60 * 1000; + const deadline = Date.now() + maxWaitMs; + + const resolutions = new Map(); + for (const workflow of criticalWorkflows) { + resolutions.set(workflow.workflowFile, { resolved: false, everFound: false }); + } + const dispatched = new Set(); + + const matchesWorkflow = (run, workflow) => { + const workflowPath = `.github/workflows/${workflow.workflowFile}`; + return ( + run.path === workflowPath || + (typeof run.path === 'string' && run.path.endsWith(`/${workflowPath}`)) || + workflow.fallbackNames.includes(run.name) + ); + }; - let branchRuns = []; - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const { data: completedRuns } = await github.rest.actions.listWorkflowRunsForRepo({ + let iteration = 0; + for (;;) { + iteration += 1; + + const { data: allRuns } = await github.rest.actions.listWorkflowRunsForRepo({ owner: context.repo.owner, repo: context.repo.repo, branch: 'nightly', - status: 'completed', per_page: 100, }); + const branchRuns = allRuns.workflow_runs; + + for (const workflow of criticalWorkflows) { + const state = resolutions.get(workflow.workflowFile); + if (state.resolved) continue; + + const matchingRuns = branchRuns + .filter((r) => r.head_sha === nightlyHeadSha && matchesWorkflow(r, workflow)) + .sort((a, b) => new Date(b.created_at) - new Date(a.created_at)); + const latestRun = matchingRuns[0]; + + if (!latestRun) { + if (workflow.dispatchable && !dispatched.has(workflow.workflowFile)) { + core.info( + `No run found for ${workflow.workflowFile} on nightly HEAD ${nightlyHeadSha}; dispatching it (iteration ${iteration})`, + ); + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: workflow.workflowFile, + ref: 'nightly', + }); + dispatched.add(workflow.workflowFile); + } else { + core.info( + `Still waiting for ${workflow.workflowFile} to appear on nightly HEAD (iteration ${iteration})`, + ); + } + continue; + } - branchRuns = completedRuns.workflow_runs; - - const allWorkflowsPresentForHead = criticalWorkflows.every((workflow) => { - const workflowPath = `.github/workflows/${workflow.workflowFile}`; - return branchRuns.some( - (r) => - r.head_sha === nightlyHeadSha && - ( - r.path === workflowPath || - (typeof r.path === 'string' && r.path.endsWith(`/${workflowPath}`)) || - workflow.fallbackNames.includes(r.name) - ), - ); - }); + state.everFound = true; + + if (latestRun.status !== 'completed') { + core.info( + `${workflow.workflowFile} is still ${latestRun.status} on nightly HEAD (iteration ${iteration})`, + ); + continue; + } + + state.resolved = true; + state.url = latestRun.html_url; + if (latestRun.conclusion === 'success') { + state.ok = true; + core.info( + `Required workflow ${workflow.workflowFile} passed for nightly HEAD via run ${latestRun.id}`, + ); + } else { + state.ok = false; + state.reason = `${workflow.workflowFile} ${latestRun.conclusion} (${latestRun.html_url})`; + core.warning( + `Required workflow ${workflow.workflowFile} is ${latestRun.conclusion} on nightly HEAD`, + ); + } + } - if (allWorkflowsPresentForHead) { - core.info(`Required workflow runs found for nightly HEAD on attempt ${attempt}`); + const allResolved = [...resolutions.values()].every((s) => s.resolved); + if (allResolved) { + core.info('All required workflows resolved for nightly HEAD'); break; } - if (attempt < maxAttempts) { - core.info( - `Waiting for required runs to appear for nightly HEAD (attempt ${attempt}/${maxAttempts})`, - ); - await new Promise((resolve) => setTimeout(resolve, waitMs)); + if (Date.now() > deadline) { + core.warning('Timed out waiting for required workflows to resolve on nightly HEAD'); + for (const workflow of criticalWorkflows) { + const state = resolutions.get(workflow.workflowFile); + if (state.resolved) continue; + + state.resolved = true; + state.ok = false; + if (!workflow.dispatchable && !state.everFound) { + // A real problem, not a timing artifact - nightly's own push + // should have triggered this workflow automatically. + state.reason = `${workflow.workflowFile} never triggered automatically for nightly HEAD ${nightlyHeadSha}`; + } else { + state.reason = `${workflow.workflowFile} timed out after 40m waiting for a completed run on nightly HEAD ${nightlyHeadSha}`; + } + } + break; } - } - if (branchRuns.length === 0) { - core.setOutput('is_healthy', 'false'); - core.setOutput('latest_run_url', 'No completed runs found'); - core.setOutput('failure_reason', 'No completed workflow runs found on nightly'); - core.warning('No completed workflow runs found on nightly - blocking promotion'); - return; + await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); } - let hasFailure = false; - let failureReason = ''; - let latestRunUrl = branchRuns[0]?.html_url || 'N/A'; + const failures = criticalWorkflows + .map((workflow) => resolutions.get(workflow.workflowFile)) + .filter((state) => state.ok === false); - for (const workflow of criticalWorkflows) { - const workflowPath = `.github/workflows/${workflow.workflowFile}`; - core.info( - `Evaluating required workflow ${workflow.workflowFile} (path match first, names fallback: ${workflow.fallbackNames.join(', ')})`, - ); - - const latestRunForHead = branchRuns.find( - (r) => - r.head_sha === nightlyHeadSha && - ( - r.path === workflowPath || - (typeof r.path === 'string' && r.path.endsWith(`/${workflowPath}`)) || - workflow.fallbackNames.includes(r.name) - ), - ); + const hasFailure = failures.length > 0; + const failureReason = failures.map((f) => f.reason).join('; '); - if (!latestRunForHead) { - hasFailure = true; - failureReason = `${workflow.workflowFile} has no completed run for nightly HEAD ${nightlyHeadSha}`; - latestRunUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/workflows/${workflow.workflowFile}`; - core.warning( - `Required workflow ${workflow.workflowFile} has no completed run for current nightly HEAD`, - ); - break; - } - - if (latestRunForHead.conclusion !== 'success') { - hasFailure = true; - failureReason = `${workflow.workflowFile} ${latestRunForHead.conclusion} (${latestRunForHead.html_url})`; - latestRunUrl = latestRunForHead.html_url; - core.warning( - `Required workflow ${workflow.workflowFile} is ${latestRunForHead.conclusion} on nightly HEAD`, - ); - break; + let latestRunUrl = 'N/A'; + if (hasFailure) { + latestRunUrl = + failures[0].url || + `https://github.com/${context.repo.owner}/${context.repo.repo}/actions`; + } else { + const anyResolvedWithUrl = [...resolutions.values()].find((s) => s.url); + if (anyResolvedWithUrl) { + latestRunUrl = anyResolvedWithUrl.url; } - - core.info( - `Required workflow ${workflow.workflowFile} passed for nightly HEAD via run ${latestRunForHead.id}`, - ); } core.setOutput('is_healthy', hasFailure ? 'false' : 'true'); @@ -452,9 +506,11 @@ jobs: core.info(`Current nightly HEAD for dispatch fallback: ${nightlyHeadSha}`); const prNumber = process.env.PR_NUMBER; + // e2e-tests-split.yml and codeql.yml are proactively dispatched (if + // missing) inside check-nightly-health itself, so re-dispatching them + // here would be redundant. These remaining workflows are supplementary + // checks not part of the health gate, dispatched once the PR exists. const requiredWorkflows = [ - { id: 'e2e-tests-split.yml' }, - { id: 'codeql.yml' }, { id: 'codecov-upload.yml', inputs: { run_backend: 'true', run_frontend: 'true' } }, { id: 'security-pr.yml', inputs: { pr_number: prNumber } }, { id: 'supply-chain-verify.yml' }, From ea81064df383800d5857b77ab0f0d88c1632ae7f Mon Sep 17 00:00:00 2001 From: Wikid82 Date: Mon, 21 Sep 2026 18:37:53 -0400 Subject: [PATCH 3/3] chore: skip propagate PR when file diff is empty despite ahead_by>0 A merge commit (e.g. the nightly -> main weekly promotion) can leave a branch "ahead" by that commit while its resulting tree is identical to the target branch, since the content already arrived there via another path. The existing ahead_by===0 guard doesn't catch this, so a no-op PR gets opened (PR #1372: 0 files changed). Skip creation when the actual file diff is empty, regardless of ahead_by. --- .github/workflows/propagate-changes.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/propagate-changes.yml b/.github/workflows/propagate-changes.yml index 30cda560c..60084f796 100644 --- a/.github/workflows/propagate-changes.yml +++ b/.github/workflows/propagate-changes.yml @@ -137,6 +137,19 @@ jobs: files = originalFiles.map(f => f.toLowerCase()); } + // ahead_by > 0 only means src has commits base lacks — it does NOT + // mean the resulting tree differs. A merge commit (e.g. the nightly -> + // main weekly promotion) can leave src "ahead" by that merge commit + // while its tree is byte-identical to base (the content already + // arrived in base via a different path), producing an empty file + // diff. Opening a PR for that is pure noise (see PR #1372: 0 files + // changed, opened solely because of the promotion merge commit) — + // skip it exactly like the ahead_by === 0 case above. + if (files.length === 0) { + core.info(`${src} -> ${base} has no file differences (ahead_by=${compare.data.ahead_by} but empty diff). No propagation needed.`); + return; + } + // Load propagation config (list of sensitive paths) from .github/propagate-config.yml when available // NOTE: .github/workflows/ was removed from defaults - workflow updates SHOULD propagate // to ensure downstream branches have correct CI/CD configurations