Main CI Failure Issue #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # .github/workflows/main-ci-failure-issue.yml | |
| name: 'Main CI Failure Issue' | |
| on: | |
| workflow_run: | |
| workflows: ['E2E Tests', 'SDK Python'] | |
| types: ['completed'] | |
| defaults: | |
| run: | |
| shell: 'bash' | |
| jobs: | |
| # Split in two so the job holding the bot PAT still checks out nothing and runs | |
| # no repository code. This job works out WHICH tests broke — the dedupe key — | |
| # from the failed run's logs, using read-only scopes and the workflow token, | |
| # and hands the finished title and body to the privileged job as outputs. | |
| analyze: | |
| name: 'Identify the failing tests' | |
| if: "${{ github.repository == 'QwenLM/qwen-code' && github.event.workflow_run.conclusion == 'failure' && github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.event == 'push' }}" | |
| runs-on: 'ubuntu-latest' | |
| timeout-minutes: 10 | |
| permissions: | |
| # Read the job logs of the triggering run. | |
| actions: 'read' | |
| contents: 'read' | |
| # Find an issue that already tracks this failure. | |
| issues: 'read' | |
| outputs: | |
| issue_number: '${{ steps.plan.outputs.issue_number }}' | |
| title: '${{ steps.plan.outputs.title }}' | |
| body: '${{ steps.plan.outputs.body }}' | |
| steps: | |
| - name: 'Checkout' | |
| uses: 'actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10' # v6.0.3 | |
| with: | |
| persist-credentials: false | |
| - name: 'Download failed job logs' | |
| env: | |
| GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}' | |
| REPO: '${{ github.repository }}' | |
| WORKFLOW_RUN_ID: '${{ github.event.workflow_run.id }}' | |
| run: |- | |
| log_dir="${RUNNER_TEMP}/failed-logs" | |
| mkdir -p "${log_dir}" | |
| mapfile -t job_ids < <( | |
| gh api "repos/${REPO}/actions/runs/${WORKFLOW_RUN_ID}/jobs?per_page=100" \ | |
| --paginate \ | |
| --jq '.jobs[] | select(.conclusion == "failure") | .id' | |
| ) | |
| echo "Failed jobs: ${#job_ids[@]}" | |
| for job_id in "${job_ids[@]}"; do | |
| if ! gh api "repos/${REPO}/actions/jobs/${job_id}/logs" \ | |
| > "${log_dir}/${job_id}.log"; then | |
| # A missing log only costs precision: with no identifiable test the | |
| # plan below falls back to the per-commit issue. | |
| echo "::warning::Could not download the log of job ${job_id}" | |
| rm -f "${log_dir}/${job_id}.log" | |
| fi | |
| done | |
| - name: 'Plan the issue' | |
| id: 'plan' | |
| env: | |
| GH_TOKEN: '${{ secrets.GITHUB_TOKEN }}' | |
| REPO: '${{ github.repository }}' | |
| WORKFLOW_NAME: '${{ github.event.workflow_run.name }}' | |
| WORKFLOW_RUN_ID: '${{ github.event.workflow_run.id }}' | |
| WORKFLOW_RUN_URL: '${{ github.event.workflow_run.html_url }}' | |
| # Actions supplies the timestamp; the helper stays free of clock reads | |
| # so its output is reproducible under test. | |
| WORKFLOW_RUN_AT: '${{ github.event.workflow_run.updated_at }}' | |
| HEAD_SHA: '${{ github.event.workflow_run.head_sha }}' | |
| run: |- | |
| shopt -s nullglob | |
| logs=("${RUNNER_TEMP}/failed-logs/"*.log) | |
| helper='.github/scripts/ci/main-failure-signature.mjs' | |
| analysis="${RUNNER_TEMP}/analysis.json" | |
| node "${helper}" analyze --workflow "${WORKFLOW_NAME}" "${logs[@]}" \ | |
| > "${analysis}" | |
| echo "Failing tests identified: $(jq '.tests | length' "${analysis}")" | |
| jq -r '.tests[].id' "${analysis}" | |
| plan_args=( | |
| --analysis "${analysis}" | |
| --sha "${HEAD_SHA}" | |
| --run-id "${WORKFLOW_RUN_ID}" | |
| --run-url "${WORKFLOW_RUN_URL}" | |
| --at "${WORKFLOW_RUN_AT}" | |
| ) | |
| plan="${RUNNER_TEMP}/plan.json" | |
| node "${helper}" plan "${plan_args[@]}" > "${plan}" | |
| # Match on any of this run's failing tests: a failure set that grew | |
| # (`[A]` then `[A, B]`) still belongs to the issue that tracks A. With | |
| # no identifiable test this is the per-commit marker instead. | |
| existing_issue='' | |
| while read -r marker; do | |
| existing_issue="$( | |
| gh issue list \ | |
| --repo "${REPO}" \ | |
| --state open \ | |
| --search "${marker} in:body" \ | |
| --json number \ | |
| --jq '.[0].number // ""' | |
| )" | |
| if [[ -n "${existing_issue}" ]]; then | |
| echo "Issue #${existing_issue} already tracks this failure (${marker})." | |
| break | |
| fi | |
| done < <(jq -r '.searchMarkers[]' "${plan}") | |
| # Re-plan against the existing body so recorded recurrences, extra | |
| # markers and hand-written notes survive. | |
| if [[ -n "${existing_issue}" ]]; then | |
| existing_body="${RUNNER_TEMP}/existing-body.md" | |
| gh issue view "${existing_issue}" \ | |
| --repo "${REPO}" \ | |
| --json body \ | |
| --jq '.body' > "${existing_body}" | |
| node "${helper}" plan "${plan_args[@]}" --existing "${existing_body}" \ | |
| > "${plan}" | |
| fi | |
| # A random delimiter keeps issue-body prose from ending the heredoc | |
| # early and injecting fresh outputs (GitHub's own hardening guidance). | |
| delim="QWEN_MAIN_CI_FAILURE_BODY_$(openssl rand -hex 16)" | |
| { | |
| echo "issue_number=${existing_issue}" | |
| echo "title=$(jq -r '.title' "${plan}")" | |
| echo "body<<${delim}" | |
| jq -r '.body' "${plan}" | |
| echo "${delim}" | |
| } >> "${GITHUB_OUTPUT}" | |
| # Every GitHub write happens here, as the autofix bot. This job deliberately | |
| # checks out nothing and runs no repository code: it only consumes the title | |
| # and body the job above produced. | |
| file_issue: | |
| name: 'Create autofix issue' | |
| needs: 'analyze' | |
| # Deliberately hosted, NOT the ECS pool: this job reports that CI broke, | |
| # and the pool being the REASON CI broke would queue the report behind | |
| # the very failure it documents. | |
| runs-on: 'ubuntu-latest' | |
| timeout-minutes: 5 | |
| permissions: | |
| issues: 'write' | |
| steps: | |
| - name: 'File or update the autofix issue' | |
| env: | |
| GH_TOKEN: '${{ secrets.CI_DEV_BOT_PAT }}' | |
| REPO: '${{ github.repository }}' | |
| EXISTING_ISSUE: '${{ needs.analyze.outputs.issue_number }}' | |
| ISSUE_TITLE: '${{ needs.analyze.outputs.title }}' | |
| ISSUE_BODY: '${{ needs.analyze.outputs.body }}' | |
| AUTOFIX_BOT: "${{ vars.AUTOFIX_BOT_LOGIN || 'qwen-code-dev-bot' }}" | |
| BUG_LABEL: 'type/bug' | |
| READY_FOR_AGENT_LABEL: 'status/ready-for-agent' | |
| AUTOFIX_APPROVED_LABEL: 'autofix/approved' | |
| run: |- | |
| apply_autofix_route() { | |
| gh issue edit "$1" \ | |
| --repo "${REPO}" \ | |
| --add-label "${BUG_LABEL},${READY_FOR_AGENT_LABEL},${AUTOFIX_APPROVED_LABEL}" \ | |
| --add-assignee "${AUTOFIX_BOT}" | |
| } | |
| body_file="${RUNNER_TEMP}/issue-body.md" | |
| printf '%s\n' "${ISSUE_BODY}" > "${body_file}" | |
| if [[ -n "${EXISTING_ISSUE}" ]]; then | |
| gh issue edit "${EXISTING_ISSUE}" \ | |
| --repo "${REPO}" \ | |
| --body-file "${body_file}" | |
| echo "Recorded this run on issue #${EXISTING_ISSUE}." | |
| apply_autofix_route "${EXISTING_ISSUE}" | |
| exit 0 | |
| fi | |
| issue_url="$( | |
| gh issue create \ | |
| --repo "${REPO}" \ | |
| --title "${ISSUE_TITLE}" \ | |
| --body-file "${body_file}" | |
| )" | |
| apply_autofix_route "${issue_url}" |