diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index e17107e38c..b803ab5d1a 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -236,6 +236,42 @@ jobs: with: path: versions-temp pattern: versions-* + - name: Scan Raw Registry Results + id: registry-raw + # Any non-zero vlt exit code is a potential outage or misconfiguration. + # This must run before `bench process`, which rewrites exit codes in + # place (dropping failed runs), so raw per-run failures are only + # visible here. + if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.partial_run != 'true' + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs') + const path = require('path') + const failures = [] + + for (const entry of fs.readdirSync('results').sort()) { + const match = /^results-(.+)-(registry-[a-z]+)$/.exec(entry) + if (!match) continue + const file = path.join('results', entry, 'benchmarks.json') + if (!fs.existsSync(file)) continue + const label = `${match[1]} ${match[2]}` + + const vlt = (JSON.parse(fs.readFileSync(file, 'utf8')).results ?? []) + .find((r) => r.command === 'vlt') + if (!vlt) { + failures.push(`- **${label}**: no vlt result`) + continue + } + const codes = vlt.exit_codes ?? [] + const bad = codes.filter((c) => c !== 0) + if (bad.length) { + failures.push(`- **${label}**: ${bad.length}/${codes.length} runs failed (exit codes: ${bad.join(', ')})`) + } + } + + core.info(failures.length ? `vlt registry run failures:\n${failures.join('\n')}` : 'all vlt registry runs succeeded') + core.setOutput('vlt_failures', failures.join('\n')) - name: Process Results run: | ./bench process @@ -272,6 +308,74 @@ jobs: name: results path: results/ retention-days: 7 + - name: Check Registry Results + id: registry-check + # Same gate as the deploy job: alert exactly when these results will + # be published to the site. + if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.partial_run != 'true' + uses: actions/github-script@v9 + env: + VLT_FAILURES: ${{ steps.registry-raw.outputs.vlt_failures }} + with: + script: | + const fs = require('fs') + const path = require('path') + // vlt must be more than 10% behind the fastest registry before a + // difference counts as noticeably slower. + const THRESHOLD = 1.1 + const slower = [] + + // Compare the processed files that get deployed to the site so an + // alert fires exactly when vlt appears slower on the site. + // Registry variations are the only ones where `command` is a + // registry name rather than a package manager. + const dir = 'results/latest' + for (const entry of fs.readdirSync(dir).sort()) { + const match = /^(.+)-(registry-[a-z]+)\.json$/.exec(entry) + if (!match) continue + const label = `${match[1]} ${match[2]}` + + // Registries where every run failed are stamped with a non-zero + // exit code and a zero mean by clean-benchmarks and rendered as + // failures on the site, so they are excluded from the speed + // comparison. A failed vlt is already reported by the raw scan. + const ok = (JSON.parse(fs.readFileSync(path.join(dir, entry), 'utf8')).results ?? []) + .filter((r) => r.mean > 0 && (r.exit_codes ?? []).every((c) => c === 0)) + const vlt = ok.find((r) => r.command === 'vlt') + if (!vlt) continue + + const best = ok.reduce((a, b) => (b.mean < a.mean ? b : a)) + if (best.command === 'vlt' || vlt.mean <= best.mean * THRESHOLD) continue + slower.push( + `- **${label}**: ${best.command} ${best.mean.toFixed(1)}s vs vlt ${vlt.mean.toFixed(1)}s (${(vlt.mean / best.mean).toFixed(2)}x slower)` + ) + } + + const failures = process.env.VLT_FAILURES ?? '' + const message = [] + if (failures) { + message.push('vlt registry runs failed (potential outage or misconfiguration):', failures) + } + if (slower.length) { + message.push(`vlt was noticeably slower than another registry in ${slower.length} registry benchmarks:`, ...slower) + } + core.info(message.length ? message.join('\n') : 'all vlt registry runs succeeded and vlt was the fastest registry') + core.setOutput('message', message.join('\n')) + - name: Slack Notify + if: steps.registry-check.outputs.message != '' + uses: rtCamp/action-slack-notify@v2.3.3 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_GITHUB_ALERTS_WEBHOOK_URL }} + SLACK_USERNAME: vltops + SLACK_CHANNEL: github-alerts + SLACK_ICON: https://github.com/vltpkg.png + SLACK_COLOR: warning + SLACK_FOOTER: "" + SLACKIFY_MARKDOWN: true + MSG_MINIMAL: "actions url,commit" + SLACK_TITLE: vlt Registry Benchmark Alert + SLACK_MESSAGE: | + ${{ steps.registry-check.outputs.message }} deploy: name: "Deploy Results" runs-on: ubuntu-24.04-arm @@ -332,3 +436,38 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: results keep_files: true + + on-failure: + name: "Notify on Failure" + runs-on: ubuntu-24.04-arm + # !cancelled() so this still runs when an upstream job failed, but stays + # quiet when the run itself was cancelled (e.g. superseded by a newer push + # via concurrency), even if a job had already failed before the cancel. + # Limited to main so branch pushes don't notify. + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && contains(needs.*.result, 'failure') }} + needs: + - detect-changes + - benchmark + - process + - deploy + - deploy-app + steps: + - name: Slack Notify + uses: rtCamp/action-slack-notify@v2.3.3 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_GITHUB_ALERTS_WEBHOOK_URL }} + SLACK_USERNAME: vltops + SLACK_CHANNEL: github-alerts + SLACK_ICON: https://github.com/vltpkg.png + SLACK_COLOR: failure + SLACK_FOOTER: "" + SLACKIFY_MARKDOWN: true + MSG_MINIMAL: "actions url,commit" + SLACK_TITLE: Benchmark Run Failed + SLACK_MESSAGE: | + The benchmark run failed and needs to be retriggered. + Detect Changes: ${{ needs.detect-changes.result }} + Benchmark: ${{ needs.benchmark.result }} + Process: ${{ needs.process.result }} + Deploy: ${{ needs.deploy.result }} + Deploy App: ${{ needs.deploy-app.result }}