diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index db35cac7fb..e17107e38c 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -7,8 +7,8 @@ on: description: "The fixture to run the benchmarks on" default: '["next", "astro", "vue", "svelte", "large", "babylon"]' variations: - description: "The benchmark variations to run" - default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "build", "build-cache", "registry-clean", "registry-lockfile"]' + description: "The benchmark variations to run (build/build-cache/run are added via matrix include pairs)" + default: '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]' binaries: description: "The binaries to run the benchmarks on" default: '"npm,yarn,berry,zpm,pnpm,pacquet,vlt,bun,deno,aube,nx,turbo,vp,node"' @@ -24,7 +24,7 @@ on: # Prevent multiple runs from interfering with each other concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-bust + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref }}-bust cancel-in-progress: true jobs: @@ -33,6 +33,7 @@ jobs: runs-on: ubuntu-24.04-arm outputs: app_only: ${{ steps.check.outputs.app_only }} + partial_run: ${{ steps.partial.outputs.partial_run }} steps: - uses: actions/checkout@v6 with: @@ -77,6 +78,30 @@ jobs: echo "app_only=$APP_ONLY" >> "$GITHUB_OUTPUT" echo "App-only changes: $APP_ONLY" + - name: Detect partial run + id: partial + env: + IN_FIXTURES: ${{ inputs.fixtures || '' }} + IN_VARIATIONS: ${{ inputs.variations || '' }} + IN_BINARIES: ${{ inputs.binaries || '' }} + IN_WARMUP: ${{ inputs.warmup || '' }} + IN_RUNS: ${{ inputs.runs || '' }} + run: | + # A dispatch with any non-default input produces a partial/tuned data + # set: results stay in artifacts + logs but must not deploy to + # gh-pages, where missing variants break chart rendering for the day. + partial=false + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + [[ "$IN_FIXTURES" == '["next", "astro", "vue", "svelte", "large", "babylon"]' ]] || partial=true + [[ "$IN_VARIATIONS" == '["clean", "node_modules", "cache", "cache+node_modules", "cache+lockfile", "cache+lockfile+node_modules", "lockfile", "lockfile+node_modules", "registry-clean", "registry-lockfile"]' ]] || partial=true + [[ "$IN_BINARIES" == '"npm,yarn,berry,zpm,pnpm,pacquet,vlt,bun,deno,aube,nx,turbo,vp,node"' ]] || partial=true + [[ "$IN_WARMUP" == "2" ]] || partial=true + [[ "$IN_RUNS" == "5" ]] || partial=true + fi + echo "partial_run=$partial" >> "$GITHUB_OUTPUT" + if [[ "$partial" == "true" ]]; then + echo "::notice::Partial run detected (non-default dispatch inputs) — gh-pages deploy will be skipped; results live in job logs and artifacts." + fi benchmark: name: "Run Benchmarks" @@ -214,6 +239,24 @@ jobs: - name: Process Results run: | ./bench process + - name: Summarize results in log + run: | + { + echo '```' + for f in results/results-*/benchmarks.json; do + [ -f "$f" ] || continue + node -e ' + const fs = require("fs") + const p = process.argv[1] + const name = p.split("/")[1].replace(/^results-/, "") + for (const r of JSON.parse(fs.readFileSync(p, "utf8")).results ?? []) { + const times = (r.times ?? []).map((t) => t.toFixed(1)).join(", ") + console.log(`${name} ${r.command}: mean=${(r.mean ?? 0).toFixed(1)}s times=[${times}]`) + } + ' "$f" + done + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" - name: Install vlt run: | npm install -g vlt@latest @@ -235,7 +278,7 @@ jobs: needs: [detect-changes, process] permissions: contents: write - if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.app_only != 'true' + if: github.ref == 'refs/heads/main' && needs.detect-changes.outputs.app_only != 'true' && needs.detect-changes.outputs.partial_run != 'true' steps: - uses: actions/checkout@v6 - name: Download Results