chore: notify on some failures

.github/workflows/benchmark.yaml

@@ -236,6 +236,42 @@ jobs:
         with:
           path: versions-temp
           pattern: versions-*
+      - name: Scan Raw Registry Results
+        id: registry-raw
+        # Any non-zero vlt exit code is a potential outage or misconfiguration.
+        # This must run before `bench process`, which rewrites exit codes in
+        # place (dropping failed runs), so raw per-run failures are only
+        # visible here.
+        if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.partial_run != 'true'
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const fs = require('fs')
+            const path = require('path')
+            const failures = []
+
+            for (const entry of fs.readdirSync('results').sort()) {
+              const match = /^results-(.+)-(registry-[a-z]+)$/.exec(entry)
+              if (!match) continue
+              const file = path.join('results', entry, 'benchmarks.json')
+              if (!fs.existsSync(file)) continue
+              const label = `${match[1]} ${match[2]}`
+
+              const vlt = (JSON.parse(fs.readFileSync(file, 'utf8')).results ?? [])
+                .find((r) => r.command === 'vlt')
+              if (!vlt) {
+                failures.push(`- **${label}**: no vlt result`)
+                continue
+              }
+              const codes = vlt.exit_codes ?? []
+              const bad = codes.filter((c) => c !== 0)
+              if (bad.length) {
+                failures.push(`- **${label}**: ${bad.length}/${codes.length} runs failed (exit codes: ${bad.join(', ')})`)
+              }
+            }
+
+            core.info(failures.length ? `vlt registry run failures:\n${failures.join('\n')}` : 'all vlt registry runs succeeded')
+            core.setOutput('vlt_failures', failures.join('\n'))
       - name: Process Results
         run: |
           ./bench process
@@ -272,6 +308,74 @@ jobs:
           name: results
           path: results/
           retention-days: 7
+      - name: Check Registry Results
+        id: registry-check
+        # Same gate as the deploy job: alert exactly when these results will
+        # be published to the site.
+        if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.partial_run != 'true'
+        uses: actions/github-script@v9
+        env:
+          VLT_FAILURES: ${{ steps.registry-raw.outputs.vlt_failures }}
+        with:
+          script: |
+            const fs = require('fs')
+            const path = require('path')
+            // vlt must be more than 10% behind the fastest registry before a
+            // difference counts as noticeably slower.
+            const THRESHOLD = 1.1
+            const slower = []
+
+            // Compare the processed files that get deployed to the site so an
+            // alert fires exactly when vlt appears slower on the site.
+            // Registry variations are the only ones where `command` is a
+            // registry name rather than a package manager.
+            const dir = 'results/latest'
+            for (const entry of fs.readdirSync(dir).sort()) {
+              const match = /^(.+)-(registry-[a-z]+)\.json$/.exec(entry)
+              if (!match) continue
+              const label = `${match[1]} ${match[2]}`
+
+              // Registries where every run failed are stamped with a non-zero
+              // exit code and a zero mean by clean-benchmarks and rendered as
+              // failures on the site, so they are excluded from the speed
+              // comparison. A failed vlt is already reported by the raw scan.
+              const ok = (JSON.parse(fs.readFileSync(path.join(dir, entry), 'utf8')).results ?? [])
+                .filter((r) => r.mean > 0 && (r.exit_codes ?? []).every((c) => c === 0))
+              const vlt = ok.find((r) => r.command === 'vlt')
+              if (!vlt) continue
+
+              const best = ok.reduce((a, b) => (b.mean < a.mean ? b : a))
+              if (best.command === 'vlt' || vlt.mean <= best.mean * THRESHOLD) continue
+              slower.push(
+                `- **${label}**: ${best.command} ${best.mean.toFixed(1)}s vs vlt ${vlt.mean.toFixed(1)}s (${(vlt.mean / best.mean).toFixed(2)}x slower)`
+              )
+            }
+
+            const failures = process.env.VLT_FAILURES ?? ''
+            const message = []
+            if (failures) {
+              message.push('vlt registry runs failed (potential outage or misconfiguration):', failures)
+            }
+            if (slower.length) {
+              message.push(`vlt was noticeably slower than another registry in ${slower.length} registry benchmarks:`, ...slower)
+            }
+            core.info(message.length ? message.join('\n') : 'all vlt registry runs succeeded and vlt was the fastest registry')
+            core.setOutput('message', message.join('\n'))
+      - name: Slack Notify
+        if: steps.registry-check.outputs.message != ''
+        uses: rtCamp/action-slack-notify@v2.3.3
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_GITHUB_ALERTS_WEBHOOK_URL }}
+          SLACK_USERNAME: vltops
+          SLACK_CHANNEL: github-alerts
+          SLACK_ICON: https://github.com/vltpkg.png
+          SLACK_COLOR: warning
+          SLACK_FOOTER: ""
+          SLACKIFY_MARKDOWN: true
+          MSG_MINIMAL: "actions url,commit"
+          SLACK_TITLE: vlt Registry Benchmark Alert
+          SLACK_MESSAGE: |
+            ${{ steps.registry-check.outputs.message }}
   deploy:
     name: "Deploy Results"
     runs-on: ubuntu-24.04-arm
@@ -332,3 +436,38 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: results
           keep_files: true
+
+  on-failure:
+    name: "Notify on Failure"
+    runs-on: ubuntu-24.04-arm
+    # !cancelled() so this still runs when an upstream job failed, but stays
+    # quiet when the run itself was cancelled (e.g. superseded by a newer push
+    # via concurrency), even if a job had already failed before the cancel.
+    # Limited to main so branch pushes don't notify.
+    if: ${{ !cancelled() && github.ref == 'refs/heads/main' && contains(needs.*.result, 'failure') }}
+    needs:
+      - detect-changes
+      - benchmark
+      - process
+      - deploy
+      - deploy-app
+    steps:
+      - name: Slack Notify
+        uses: rtCamp/action-slack-notify@v2.3.3
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_GITHUB_ALERTS_WEBHOOK_URL }}
+          SLACK_USERNAME: vltops
+          SLACK_CHANNEL: github-alerts
+          SLACK_ICON: https://github.com/vltpkg.png
+          SLACK_COLOR: failure
+          SLACK_FOOTER: ""
+          SLACKIFY_MARKDOWN: true
+          MSG_MINIMAL: "actions url,commit"
+          SLACK_TITLE: Benchmark Run Failed
+          SLACK_MESSAGE: |
+            The benchmark run failed and needs to be retriggered.
+            Detect Changes: ${{ needs.detect-changes.result }}
+            Benchmark: ${{ needs.benchmark.result }}
+            Process: ${{ needs.process.result }}
+            Deploy: ${{ needs.deploy.result }}
+            Deploy App: ${{ needs.deploy-app.result }}