docs(benchmarking): Add Sonnet 5 Pi benchmark

packages/docs/src/components/BenchmarkRuns.astro

@@ -36,6 +36,7 @@ type BenchmarkResult = {
     findingsTotal: number;
     inputTokens?: number;
     outputTokens?: number;
+    skippedFileCount?: number;
   };
   supersededBy?: string;
   targetMode: string;
@@ -91,6 +92,9 @@ const modelLabel = (model: string) => {
   if (model === "anthropic/claude-sonnet-4-6") {
     return "Claude Sonnet 4.6";
   }
+  if (model === "anthropic/claude-sonnet-5") {
+    return "Claude Sonnet 5";
+  }
   if (model === "claude-sonnet-4-6") {
     return "Claude Sonnet 4.6";
   }
@@ -191,6 +195,7 @@ const stableResults = Object.values(resultModules)
       result.targetMode === "all-corpus-files-by-sha" &&
       !result.supersededBy &&
       result.summary.chunksFailed === 0 &&
+      (result.summary.skippedFileCount ?? 0) === 0 &&
       result.summary.chunksAnalyzed === result.summary.chunksTotal &&
       result.timing?.analysisChunkMs,
   );

packages/docs/src/content/docs/benchmarking.mdx

@@ -52,15 +52,38 @@ hidden here.
   normalized model pricing or cost per finding. **P50** and **P90** are
   per-analysis-chunk durations. **Total** includes verifier work, provider
   latency, queueing, retries, and runtime overhead.
-- The stable matrix only shows clean comparison runs: 156 analysis chunks, zero
-  failed chunks, and per-chunk timing metadata. Partial and superseded rows stay
-  in the result data for audit history.
+- The stable matrix only shows clean comparison runs: complete target-file
+  coverage, zero failed chunks, and per-chunk timing metadata. Partial and
+  superseded rows stay in the result data for audit history.
 - Trace and auxiliary usage fields depend on what the raw artifacts preserved.
   When verifier usage is available, it appears under
   `auxiliaryUsage.verification`.
 
 ## Analysis
 
+### Sonnet 5 High on Pi
+
+The Sonnet 5 high-effort Pi row uses Warden's workspace Pi dependencies, not a
+globally installed Pi binary. The benchmark ran with
+`@earendil-works/pi-ai`/`@earendil-works/pi-coding-agent` 0.80.3, which exposes
+`anthropic/claude-sonnet-5`. The direct Anthropic key path worked, so the run
+did not use the OpenRouter fallback.
+
+Sonnet 5 high found 22 of 86 known corpus entries and emitted 27 final
+findings. It scans all 79 target files as 156 traced chunks, with zero failed
+chunks. The run cost $23.46 total, including $15.38 of scan work and $8.08 of
+auxiliary verifier and merge work.
+
+The practical read is that Sonnet 5 high is competitive but not better than
+Sonnet 4.6 high on this corpus. It trails the Sonnet 4.6 Pi row by three known
+matches, trails DeepSeek V4 Pro xhigh and Opus 4.6 high by one, and lands one
+known match above Opus 4.8 high. Sonnet 5 also costs more than Sonnet 4.6 on Pi
+($23.46 versus $19.84), while emitting fewer final findings than Sonnet 4.6
+(27 versus 32). It finds several different classes of issues in the `7f41cc50`
+shard, including OAuth redirect prefix matching, Seer RPC HMAC path binding,
+and frontend tooltip XSS, but misses enough other corpus matches that the final
+score is lower than Sonnet 4.6.
+
 ### Sonnet 4.6: Claude SDK vs Pi
 
 The Sonnet 4.6 comparison is clean enough to compare directly. Both rows scan

packages/docs/src/content/docs/benchmarking/running.mdx

@@ -41,6 +41,13 @@ full OpenRouter selector, such as `openrouter/deepseek/deepseek-v4-pro`.
 For the Claude SDK runtime, use Claude Code model IDs such as
 `claude-sonnet-4-6` instead of Pi provider/model selectors.
 
+Warden uses its workspace Pi dependencies, not a globally installed `pi`
+binary. If a newly released model is missing from the Pi registry, update
+`@earendil-works/pi-ai` and `@earendil-works/pi-coding-agent` in Warden before
+running the benchmark. Claude Sonnet 5 is available in Pi `0.80.3` as
+`anthropic/claude-sonnet-5`; through OpenRouter use
+`openrouter/anthropic/claude-sonnet-5`.
+
 If you load credentials from a local env file, export them before running
 `pnpm`. A plain `source .env.local` sets shell variables, but child processes
 will not see them unless they are exported:
@@ -116,6 +123,10 @@ version = 1
 reportOn = "low"
 maxTurns = 100
 
+[defaults.scan]
+maxFiles = 200
+maxChangedLines = 50000
+
 [defaults.verification]
 enabled = true
 
@@ -131,6 +142,15 @@ Set `maxTurns` explicitly for corpus runs. The default is lower and can make
 heavy security chunks fail with `turn_limit`, which turns a model comparison
 into a runner-limit comparison.
 
+Set scan limits explicitly too. Corpus benchmarks pass an explicit target-file
+list and must not silently skip those targets because a normal PR scan budget
+was exceeded.
+
+Action item: explicit local file-target runs should not apply the same
+changed-line budget as PR-wide scans. The benchmark uses higher scan limits
+until the local CLI distinguishes those modes. Track this in
+[getsentry/warden#419](https://github.com/getsentry/warden/issues/419).
+
 ## Run
 
 Run from the Warden repository:
@@ -200,6 +220,9 @@ const summary = records.find((record) => record.type === "summary");
 const sourceChunks = records.filter(
   (record) => record.status === "ok" && record.chunk?.file,
 );
+const skippedFiles = records
+  .filter((record) => record.status === "skipped" && record.chunk?.file)
+  .flatMap((record) => record.skippedFiles?.map((file) => file.filename) ?? [record.chunk.file]);
 const missingTraces = sourceChunks.filter((record) => !record.trace);
 const chunksByFile = new Map();
 for (const record of sourceChunks) {
@@ -214,6 +237,9 @@ if (failed.length > 0) {
 if (!summary) {
   throw new Error(`${output} is missing a summary record`);
 }
+if (skippedFiles.length > 0 || (summary.totalSkippedFiles ?? 0) > 0) {
+  throw new Error(`${output} does not have complete target-file coverage`);
+}
 const expectedFileSet = new Set(expectedFiles);
 const missingFiles = expectedFiles.filter((file) => !chunksByFile.has(file));
 const unexpectedFiles = [...chunksByFile.keys()].filter(