diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index a9b69c265..eb3605a05 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -22,10 +22,6 @@ on: description: "Run deep smoke tests for the AI search benchmark template" type: boolean default: true - run_solvability: - description: "Run ScaffBench per-spec solvability gate (all ecosystems)" - type: boolean - default: true permissions: contents: read @@ -399,69 +395,6 @@ jobs: if-no-files-found: warn retention-days: 7 - scaffbench-solvability: - name: ScaffBench Spec Solvability - if: ${{ github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.run_solvability) }} - runs-on: ubuntu-latest - timeout-minutes: 90 - steps: - - uses: actions/checkout@v4 - - - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - uses: actions/setup-node@v4 - with: - node-version: "22" - - - uses: astral-sh/setup-uv@v5 - - - uses: erlef/setup-beam@v1 - with: - otp-version: "27" - elixir-version: "1.18" - - - uses: actions/setup-dotnet@v4 - with: - dotnet-version: "10.0.x" - - - name: Install protoc - # The rust-leptos-axum spec uses Tonic/gRPC, whose prost build script - # compiles .proto files with protoc at `cargo check` time. - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - - name: Show toolchains - run: | - echo "cargo: $(cargo --version 2>&1 || echo MISSING)" - echo "go: $(go version 2>&1 || echo MISSING)" - echo "uv: $(uv --version 2>&1 || echo MISSING)" - echo "dotnet: $(dotnet --version 2>&1 || echo MISSING)" - echo "elixir: $(elixir --version 2>&1 || echo MISSING)" - echo "mix: $(mix --version 2>&1 || echo MISSING)" - echo "protoc: $(protoc --version 2>&1 || echo MISSING)" - - - uses: actions/cache@v4 - with: - path: | - node_modules - ~/.bun/install/cache - key: solvability-${{ runner.os }}-bun-${{ hashFiles('bun.lock') }} - restore-keys: solvability-${{ runner.os }}-bun- - - - name: Install Dependencies - run: bun install --frozen-lockfile - - - name: Build Packages - run: | - bun run --cwd packages/types build - bun run --cwd packages/project-lifecycle build - bun run --cwd packages/template-generator build - bun run --cwd apps/cli build - - - name: Run Spec Solvability Gate - run: bun test apps/cli/test/e2e/scaffbench-solvability.test.ts --timeout 1200000 - playwright: name: Playwright Web Builder Tests if: ${{ github.event_name != 'workflow_dispatch' || inputs.run_playwright }} diff --git a/.gitignore b/.gitignore index 908ee7e26..4fb378738 100644 --- a/.gitignore +++ b/.gitignore @@ -52,7 +52,6 @@ yarn.lock .smoke-manifest.json testing/.release-guard/ testing/.published-package/ -testing/.tmp-scaffbench-*/ reports/ artifacts/demos/ COMPARISON-PLAN.md diff --git a/AGENTS.md b/AGENTS.md index b60b8074e..9181e6d41 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,7 +27,6 @@ See `docs/guidelines/` for deeper reference on these topics: - `template-output-and-validation.md` - template conditional logic, generated output validation, sync test discipline, and framework-specific constraints - `remotion-video-style.md` - default visual style, color system, motion rules, and branding for Remotion videos in this project - `design-reading-guide.md` - ordered index of design-related markdown (agent skills + BF video style), precedence when sources conflict, and commands to verify coverage -- `scaffbench-benchmark.md` - ScaffBench protocol, execution, validation, and publication rules - `adding-new-tool-options/` - **read this subfolder when adding any new library, tool, or category** to any ecosystem (TypeScript, Rust, Go, Python). Covers every file that must be touched, with worked examples, template handler reference, test patterns, and routing edge cases (Convex skips, self-backend, frontend array detection, processor ordering) ## Web UI diff --git a/README.md b/README.md index 0454d4752..93239e010 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Connect your AI coding agents to Better Fullstack with one command: npx create-better-fullstack@latest install ``` -After install, supported agents scaffold projects through the MCP server, 2.6x faster than hand-writing in ScaffBench; see the [AI docs](https://better-fullstack.dev/docs/ai/overview) for setup. +After install, supported agents scaffold projects through the MCP server; see the [AI docs](https://better-fullstack.dev/docs/ai/overview) for setup.
diff --git a/apps/cli/test/benchmarks/scaffbench-solvability-contract.test.ts b/apps/cli/test/benchmarks/scaffbench-solvability-contract.test.ts deleted file mode 100644 index 44452510d..000000000 --- a/apps/cli/test/benchmarks/scaffbench-solvability-contract.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from "bun:test"; -import { join } from "node:path"; - -const repositoryPath = (...segments: string[]) => - join(import.meta.dir, "..", "..", "..", "..", ...segments); - -describe("ScaffBench solvability CI contract", () => { - it("provisions Beam for the Elixir ecosystem", async () => { - const workflow = Bun.YAML.parse( - await Bun.file(repositoryPath(".github", "workflows", "e2e-test.yaml")).text(), - ) as { - jobs?: Record }>; - }; - const steps = workflow.jobs?.["scaffbench-solvability"]?.steps ?? []; - - expect(steps.some((step) => step.uses === "erlef/setup-beam@v1")).toBe(true); - }); - - it("allows local skips but makes missing CI toolchains fatal", async () => { - const source = await Bun.file( - repositoryPath("apps", "cli", "test", "e2e", "scaffbench-solvability.test.ts"), - ).text(); - - expect(source).toContain("missing.length > 0 && !process.env.CI ? it.skip : it"); - expect(source).toContain("CI is missing required toolchain(s)"); - expect(source).toContain(").toEqual([])"); - }); -}); diff --git a/apps/cli/test/e2e/scaffbench-solvability.test.ts b/apps/cli/test/e2e/scaffbench-solvability.test.ts deleted file mode 100644 index 3f3843459..000000000 --- a/apps/cli/test/e2e/scaffbench-solvability.test.ts +++ /dev/null @@ -1,142 +0,0 @@ -import * as BunContext from "@effect/platform-bun/BunContext"; -import { - isAdvisoryStep, - parseArgs, - SCAFFBENCH_2_SPECS, - validateProject, - type BenchmarkSpec, -} from "@scaffbench/index"; -import { scaffoldWithCLIBinary } from "@test/e2e/e2e-utils"; -import { afterAll, beforeAll, describe, expect, it } from "bun:test"; -import * as Effect from "effect/Effect"; -import { mkdir, rm } from "node:fs/promises"; -import { join } from "node:path"; - -const SMOKE_DIR = join(import.meta.dir, "..", "..", ".smoke-scaffbench-solvability"); -const CLI_BINARY_PATH = join(import.meta.dir, "..", "..", "dist", "cli.mjs"); -const SCAFFOLD_TIMEOUT_MS = 300_000; -const TEST_TIMEOUT_MS = 1_200_000; - -const SPEC_TOOLCHAINS: Record = { - "ai-search-workbench": [], - "rust-leptos-axum": ["cargo"], - "python-ingestion-api": ["uv"], - "go-realtime-api": ["go"], - "multi-dotnet-ops": ["dotnet"], - "ts-svelte-edge-orpc": [], - "dotnet-blazor-cqrs": ["dotnet"], - "multi-ts-go-grpc": ["go"], - "java-spring-jooq-keycloak": ["mvn"], - "elixir-broadway-absinthe": ["mix"], - "react-native-expo": [], -}; - -const EXPECTED_FILE_BY_FAMILY: Record = { - typescript: "package.json", - "multi-ecosystem": "package.json", - rust: "Cargo.toml", - python: "pyproject.toml", - go: "go.mod", - java: "pom.xml", - elixir: "mix.exs", - "react-native": "package.json", -}; - -function selectSpecs(): BenchmarkSpec[] { - const filter = process.env.SCAFFBENCH_SOLVABILITY_SPECS?.split(",") - .map((value) => value.trim()) - .filter(Boolean); - return SCAFFBENCH_2_SPECS.filter((spec) => spec.supportedByBetterFullstack).filter( - (spec) => !filter?.length || filter.includes(spec.id), - ); -} - -function runValidation(spec: BenchmarkSpec, projectDir: string) { - return Effect.runPromise( - validateProject(spec, projectDir, parseArgs([])).pipe(Effect.provide(BunContext.layer)), - ); -} - -describe("ScaffBench 2 spec solvability", () => { - beforeAll(async () => { - await rm(SMOKE_DIR, { recursive: true, force: true }); - await mkdir(SMOKE_DIR, { recursive: true }); - }); - - afterAll(async () => { - if (!process.env.CI) { - await rm(SMOKE_DIR, { recursive: true, force: true }); - } - }); - - it("executes the ScaffBench validation Effect at the test boundary", async () => { - const emptyProjectDir = join(SMOKE_DIR, "effect-execution-contract"); - await mkdir(emptyProjectDir, { recursive: true }); - - const validation = await runValidation(SCAFFBENCH_2_SPECS[0]!, emptyProjectDir); - - expect(Effect.isEffect(validation)).toBe(false); - expect(validation.steps["unvalidated:project"]?.status).toBe("ran"); - expect(validation.steps["unvalidated:project"]?.exitCode).toBe(1); - }); - - for (const spec of selectSpecs()) { - const missing = (SPEC_TOOLCHAINS[spec.id] ?? []).filter((tool) => !Bun.which(tool)); - const register = missing.length > 0 && !process.env.CI ? it.skip : it; - - if (missing.length > 0 && !process.env.CI) { - console.warn( - `[scaffbench-solvability] SKIP ${spec.id}: missing toolchain(s) ${missing.join(", ")}`, - ); - } - - register( - `scaffolds and validates the ${spec.id} stack from its own canonical flags`, - async () => { - expect( - missing, - `CI is missing required toolchain(s) for ${spec.id}: ${missing.join(", ")}`, - ).toEqual([]); - const projectDir = join(SMOKE_DIR, spec.id); - const expectedFile = EXPECTED_FILE_BY_FAMILY[spec.family]; - - const scaffold = await scaffoldWithCLIBinary(projectDir, [...spec.canonicalFlags], { - cliPath: CLI_BINARY_PATH, - timeout: SCAFFOLD_TIMEOUT_MS, - expectedFiles: expectedFile ? [expectedFile] : [], - }); - expect(scaffold.ok, `scaffold failed for ${spec.id}: ${scaffold.stderrTail ?? ""}`).toBe( - true, - ); - - const validation = await runValidation(spec, projectDir); - - const coreSteps = Object.entries(validation.steps).filter( - ([name, step]) => step && !isAdvisoryStep(name) && step.status !== "na", - ); - expect( - coreSteps.length, - `no core validation step ran for ${spec.id}, the scaffold produced no recognizable project (likely a missing flag left an interactive prompt)`, - ).toBeGreaterThan(0); - - const failures = Object.entries(validation.steps) - .filter(([name, step]) => step && !isAdvisoryStep(name) && step.status !== "na") - .filter(([, step]) => step!.exitCode !== 0 || step!.timedOut) - .map(([name, step]) => ({ - name, - command: step!.command, - exitCode: step!.exitCode, - timedOut: step!.timedOut, - spawnError: step!.spawnError ?? false, - stderrTail: step!.stderrTail?.slice(-1000), - })); - - expect( - failures, - `solvability validation failed for ${spec.id}:\n${JSON.stringify(failures, null, 2)}`, - ).toEqual([]); - }, - TEST_TIMEOUT_MS, - ); - } -}); diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.json index 4ef2d5dbb..2128fe2bc 100644 --- a/apps/cli/tsconfig.json +++ b/apps/cli/tsconfig.json @@ -12,7 +12,6 @@ "types": ["node"], "paths": { "@/*": ["./src/*"], - "@scaffbench/*": ["../../scripts/scaffbench/*"], "@test/*": ["./test/*"], "@testing/*": ["../../testing/*"], "@web/*": ["../web/src/*"] diff --git a/apps/web/content/docs/verification.mdx b/apps/web/content/docs/verification.mdx index fc7fbd9d6..b50e1425e 100644 --- a/apps/web/content/docs/verification.mdx +++ b/apps/web/content/docs/verification.mdx @@ -27,7 +27,7 @@ The builder and stack pages read the option-level projection from recipe remain listed and show their limitation. A template, schema, dependency, toolchain, or deployed-commit change needs a new receipt. -## Separate from ScaffBench +## Separate from Fixproof -ScaffBench evaluates how coding models respond to scaffold tasks. It does not verify a Better +Fixproof evaluates coding agents on sealed issues with hidden tests. It does not verify a Better Fullstack release, and it does not raise the evidence level shown here. diff --git a/apps/web/messages/de.json b/apps/web/messages/de.json index e39a733f8..7caf8341a 100644 --- a/apps/web/messages/de.json +++ b/apps/web/messages/de.json @@ -132,8 +132,8 @@ "changelogAllReleases": "Alle Veröffentlichungen", "changelogLatest": "Neueste", "changelogRelease20260612Title": "Agent-Benchmark, .NET-Ökosystem und eine um 42 % leichtere Installation", - "changelogRelease20260612Summary": "Diese Version misst, wie AI-Agenten mit Better Fullstack ein Gerüst bilden, veröffentlicht die Ergebnisse auf der Homepage, fügt .NET als erstklassiges Ökosystem zum neuen Stapeldiagramm hinzu und liefert eine viel schlankere Installation. Es behebt außerdem vier Gerüstfehler, die der Benchmark selbst aufgedeckt hat.", - "changelogRelease20260612HighlightBenchmark": "Wir haben Frontier-Modelle beim Erstellen derselben Projektspezifikationen auf drei Wegen verglichen: nur per Prompt, über unser CLI und über unseren MCP-Server. Agenten auf dem MCP-Pfad waren bis zu 7× schneller bei 4× weniger Ausgabetokens; die vollständigen Ergebnisse gibt es live auf der Homepage mit einem interaktiven Diagramm.", + "changelogRelease20260612Summary": "Diese Version misst, wie AI-Agenten mit Better Fullstack ein Gerüst bilden, veröffentlicht die Ergebnisse auf der Benchmark-Seite, fügt .NET als erstklassiges Ökosystem zum neuen Stapeldiagramm hinzu und liefert eine viel schlankere Installation. Es behebt außerdem vier Gerüstfehler, die der Benchmark selbst aufgedeckt hat.", + "changelogRelease20260612HighlightBenchmark": "Wir haben Frontier-Modelle beim Erstellen derselben Projektspezifikationen auf drei Wegen verglichen: nur per Prompt, über unser CLI und über unseren MCP-Server. Agenten auf dem MCP-Pfad waren bis zu 7× schneller bei 4× weniger Ausgabetokens; die vollständigen Ergebnisse wurden damals mit einem interaktiven Diagramm auf der Benchmark-Seite veröffentlicht.", "changelogRelease20260612HighlightMcp": "Die MCP-Seite wurde mit One-Paste-Setup für Claude Code, Codex, Gemini CLI, Cursor, VS Code, Claude Desktop, Windsurf und Zed neu gestaltet.", "changelogRelease20260612HighlightDotnet": ".NET als erstklassiges Ökosystem sowie eine Unternehmensebene, Backend-Utils und Render/Netlify-Bereitstellungsoptionen im Stapeldiagramm hinzugefügt.", "changelogRelease20260612HighlightInstall": "Installationsgröße um 42 % reduziert (122 MB → 71 MB) und den Web-Entry-Chunk um 32 %.", @@ -211,16 +211,15 @@ "mcpWorkflowTitleB": "Es baut.", "mcpWorkflowDescription": "Agenten verwenden dasselbe Schema und dieselben Kompatibilitätsregeln wie der Web Builder, sodass die generierten Befehle mit dem übereinstimmen, was Benutzer visuell konfigurieren können.", "mcpTerminalHeader": "Agentensitzung", - "mcpFinalEyebrow": "Benchmark-gestützt", - "mcpFinalTitle": "2,6× schneller als", - "mcpFinalTitleEmphasis": "Nur per Prompt.", - "mcpFinalDescription": "In ScaffBench ist die MCP-geführte Projekterstellung schneller und zuverlässiger, als einen Agenten zu bitten, ein Projekt von Grund auf handschriftlich zu schreiben.", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "Sehen Sie, wie Agenten abschneiden bei", + "mcpFinalTitleEmphasis": "echten Fehlern.", + "mcpFinalDescription": "Fixproof bewertet Coding-Agenten an versiegelten, echten Fehlern aus privaten und öffentlichen Codebasen, verifiziert durch verborgene Tests.", "mcpViewBenchmark": "Benchmark ansehen", "mcpReadDocs": "Lesen Sie die Dokumentation zu MCP", "mcpStatStructuredTools": "strukturierte Werkzeuge", "mcpStatReadableResources": "lesbare Ressourcen", "mcpStatConfigurableOptions": "konfigurierbare Optionen", - "mcpStatFasterPromptOnly": "schneller als nur per Prompt", "mcpCopyAgentConfiguration": "Kopieren Sie die {agent}-Konfiguration", "mcpToolGuidanceDescription": "Workflow-Regeln, Feldsemantik und kritische Einschränkungen", "mcpToolSchemaDescription": "Gültige Optionen für jede Kategorie, filterbar nach Ökosystem", @@ -245,34 +244,6 @@ "mcpWorkflowCreateNote": "geschrieben nach ./my-app", "mcpWorkflowDoneName": "Gerüst fertig", "mcpWorkflowDoneNote": "Führen Sie bun install aus, um den Vorgang abzuschließen", - "llmBenchmarkDescription": "Messung von Codierungsagenten bei echten Fullstack-Scaffolding-Aufgaben – Zeit, Token, Kosten und ob das Ergebnis tatsächlich erstellt wird.", - "llmReadBlog": "Lesen Sie den Blog", - "llmTryMcp": "Probieren Sie MCP aus", - "llmBenchmarkMetric": "Benchmark-Metrik", - "llmScatterAria": "Benchmark-Streudiagramm: Jeder Punkt ist ein Modell und ein Erstellungspfad", - "llmScatterUnmetered": "Auf dieser Achse nicht gemessen (nicht im Diagramm): {models}", - "llmFilterModels": "Modelle filtern", - "llmModels": "Modelle", - "llmClaudeSweep": "12. Juni Sweep", - "llmCodexSweep": "10. Juni Sweep", - "llmLightSweep": "12. Juni leichter Sweep", - "llmPathCliShort": "BF-Erwähnung", - "llmPathPromptShort": "Prompt", - "llmPathMcpDetail": "erstellt Gerüste über unsere MCP-Tools", - "llmPathCliDetail": "Der Agent erstellt den Befehl Better-Fullstack CLI", - "llmPathPromptDetail": "kein Better-Fullstack – der Agent schreibt jede Datei von Hand", - "llmBuildsPassing": "Erfolgreiche Builds", - "llmAvgScaffoldTime": "Durchschnittliche Gerüstzeit", - "llmOutputTokens": "Ausgabetoken pro Gerüst", - "llmFailedBuilds": "Fehlgeschlagene Builds", - "llmSpeed": "Geschwindigkeit", - "llmTokens": "Token", - "llmErrorRate": "Fehlerquote", - "llmMostEfficient": "am effizientesten ↗", - "llmFastReliable": "schnell + zuverlässig ↗", - "llmAgentTitle": "Geben Sie Ihrem Agenten den schnellen Weg.", - "llmAgentDescription": "Ein MCP-Server, jedes vom Benchmark verwendete Spec-to-Scaffold-Tool. Wählen Sie Ihren Agenten aus, fügen Sie ihn ein, fertig.", - "llmAllSupportedClients": "alle unterstützten Clients", "llmCopyAgentSetupCommand": "Kopieren Sie den Setup-Befehl {agent}", "llmRunInTerminal": "in Ihrem Terminal ausführen", "llmPasteInto": "einfügen in {target}", @@ -473,56 +444,7 @@ "actionsReset": "Zurücksetzen", "actionsRandom": "Zufällig", "navAnalytics": "Analysen", - "runSeoTitle": "Betreibe ScaffBench selbst", - "runSeoDescription": "Reproduzieren Sie den ScaffBench-Benchmark lokal: Klonen Sie das Harness, verweisen Sie es auf eine beliebige Agenten-CLI oder einen API-Schlüssel und bewerten Sie, ob die generierten Projekte kompiliert werden können.", - "runHeroEyebrow": "Reproduziere es", - "runHeroTitleA": "Run ScaffBench", - "runHeroTitleB": "selbst", - "runHeroDescription": "Das Framework ist Open Source. Klonen Sie es, verweisen Sie es auf einen beliebigen Agenten – Claude Code, Codex, opencode, Kilo oder Antigravity für Gemini – und es generiert für jede Spezifikation ein Gerüst und prüft anschließend, ob das erstellte Projekt installiert und kompiliert werden kann. Die Ausführung erfolgt über eine angemeldete CLI oder einen einfachen API-Schlüssel.", - "runHeroQuickstart": "Schnellstart", - "runHeroBrowseReports": "Durchsuchen Sie die Berichte.", - "runQuickstartEyebrow": "Schnellstart", - "runQuickstartTitle": "Drei Schritte", - "runStepCloneInstall": "Klonen und installieren", - "runStepAuth": "Authentifizieren Sie Ihren Agenten", - "runStepRun": "Führen Sie den Benchmark aus.", - "runLabelClone": "Klonen Sie das Harness", - "runLabelSignin": "Melden Sie sich bei Ihrem Agenten an.", - "runLabelExportKey": "einen Anbieterschlüssel exportieren", - "runLabelRunAll": "Führe alle 13 Spezifikationen aus, Eingabeaufforderungspfad", - "runLabelTwoPhase": "zweiphasig", - "runAuthCliTab": "Angemeldete CLI", - "runAuthApiTab": "API-Schlüssel", - "runAuthCliDesc": "Verwenden Sie eine Agenten-CLI, bei der Sie bereits angemeldet sind (Abonnement/OAuth). Melden Sie sich einmal an, danach übernimmt das Framework die Steuerung – es sind keine Schlüssel in Ihrer Umgebung erforderlich.", - "runAuthApiDesc": "Bevorzugen Sie einen API-Schlüssel? Exportieren Sie den Anbieterschlüssel, und die gleiche Agenten-CLI wird darüber abgerechnet – ein Abonnement ist nicht erforderlich. Wir veröffentlichen abonnementbasierte Ausführungen; API-Ausführungen sind ungetestet, werden aber unterstützt.", - "runTwoPhaseNote": "Sie möchten die Validierung übersichtlich halten? Teilen Sie sie in zwei Phasen auf – generieren Sie zuerst alles und validieren Sie anschließend separat:", - "runResultsNotePre": "Die Ergebnisse – eine Rangliste, die Ergebnisse pro Spezifikation, die verwendeten Bibliotheken und die Kosten – landen im Ausgabeverzeichnis, in der gleichen Struktur wie die ", - "runResultsNoteLink": "veröffentlichte Berichte", - "runAgentsEyebrow": "Agenten & Models", - "runAgentsTitle": "Bringen Sie jeden beliebigen Agenten mit.", - "runAgentsDesc": "Der Provider wird aus der Modell-ID abgeleitet, sodass ein Flag sowohl das Modell als auch die zugehörige CLI auswählt.", - "runColAgent": "Agent", - "runColModels": "Beispielmodelle", - "runColAuth": "Authentifizierung", - "runFlagsEyebrow": "Flags", - "runFlagsTitle": "Den Lauf optimieren", - "runFlagModel": "Das auszuführende Modell (siehe Tabelle oben); der Provider wird aus der ID abgeleitet.", - "runFlagEfforts": "Argumentationsaufwand, sofern das Modell dies unterstützt", - "runFlagPaths": "Die Eingabeaufforderung schreibt alles manuell; MCP durchläuft die MCP-Tools; CLI setzt den CLI-Befehl zusammen.", - "runFlagSpecs": "Standardmäßig die vollständige Suite mit 13 Spezifikationen oder eine durch Kommas getrennte Teilmenge der Spezifikations-IDs.", - "runFlagPhase": "Der Lauf wird in eine Generierungsphase und eine Validierungsphase unterteilt, die jeweils selbstständig validiert wird.", - "runFlagOutDir": "wo die Ergebnisse landen; verwenden Sie dasselbe Verzeichnis wieder, um fortzufahren oder zu validieren", - "runCtaEyebrow": "Vergleichen", - "runCtaTitle": "Sehen Sie, wie Ihr Durchlauf abschneidet", - "runCtaDesc": "Ihre Zahlen werden im gleichen Format wie die Rangliste angezeigt. Haben Sie etwas Interessantes ausprobiert? Erstellen Sie einen Pull Request mit Ihrem Bericht.", - "runCtaLeaderboard": "Rangliste ansehen", - "llmRunItYourself": "Führen Sie es selbst aus", "navBenchmark": "Benchmark", - "benchmarkTeaserTitle": "Wie gut bauen KI-Modelle deine Projekte?", - "benchmarkTeaserTopModels": "Top-Modelle", - "benchmarkTeaserCta": "Vollständigen Benchmark ansehen", - "benchmarkSeoTitle": "ScaffBench - Wie gut bauen KI-Modelle deine Projekte?", - "benchmarkTeaserMcpBody": "Der Unterschied ist unser MCP. Richte einen beliebigen Coding-Agenten auf die Tools von Better-Fullstack und selbst ein kleines kostenloses Modell baut fast alles – mit einem Bruchteil der Tokens und Schritte.", "navUpdates": "Updates", "navTemplates": "Vorlagen", "builderNewFilter": "Neu in diesem Release", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "Full Stack", "homeStarterShapeFrontend": "Nur Frontend", "homeStarterShapeBackend": "Nur Backend", - "homeStarterShapeMobile": "Mobile App" + "homeStarterShapeMobile": "Mobile App", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof bewertet Coding-Agenten an versiegelten, echten Fehlern aus privaten und öffentlichen Codebasen, verifiziert durch verborgene Tests.", + "fixproofSeoTitle": "Fixproof: versiegelter Benchmark für Coding-Agenten", + "fixproofClaim": "Echte Fehler aus privaten und öffentlichen Codebasen, versiegelt. Verborgene Tests entscheiden.", + "fixproofProvenanceSummary": "Jede Zahl stammt aus einem aufgezeichneten, unbeaufsichtigten Lauf des genannten Agent-CLI auf einer eigenen Linux-Benchmaschine, gegen den Basis-Commit der Aufgabe und bewertet von verborgenen Tests, die bei diesem Commit nachweislich rot und mit dem Fix der Maintainer grün waren. Nichts hier ist von Hand bewertet.", + "fixproofStatusRunLabel": "Testlauf", + "fixproofStatusDateLabel": "Datum", + "fixproofStatusGradedLabel": "Bewertete Aufgaben", + "fixproofStatusTrialsLabel": "Versuche pro Aufgabe", + "fixproofGradedOfTotal": "{graded} von {total}", + "fixproofBoardHeading": "Rangliste", + "fixproofBoardCaption": "Eine Zeile pro Modell. Sortierbar nach beiden Indizes. Das Fragezeichen an einer Spalte erklärt, was sie zählt.", + "fixproofColModel": "Modell", + "fixproofColHarness": "Harness", + "fixproofColEffort": "Effort", + "fixproofColResolvedIndex": "Resolved-Index", + "fixproofColProgressIndex": "Progress-Index", + "fixproofColSolvedOverGraded": "Gelöst / bewertet", + "fixproofColRegressions": "Regressionen", + "fixproofColTestEdits": "Test-Änderungen zurückgesetzt", + "fixproofColClaimedOnly": "Behauptet, nicht erledigt", + "fixproofColMedianMinutes": "Median-Minuten", + "fixproofColRunDate": "Laufdatum", + "fixproofColTrials": "Versuche", + "fixproofDefHarness": "Das Agenten-CLI, das das Modell gesteuert hat.", + "fixproofDefResolvedIndex": "Nach Schwierigkeit gewichteter Anteil der Aufgaben, bei denen jede verborgene Prüfung bestanden wurde und keine Regression auftrat. Das ist die zentrale Kennzahl.", + "fixproofDefProgressIndex": "Gewichteter Anteil der Anforderungen einer Aufgabe, die auf dem Basis-Commit fehlgeschlagen sind und nach dem Patch des Agenten bestehen, anschließend über alle Aufgaben nach Schwierigkeit gewichtet. Jede Anforderung hat ein Gewicht von 2 (Kern), 1 oder 0,5 (Rand); Anforderungen, die auf dem Basis-Commit schon grün waren, sowie ungetestete Anforderungen werden sowohl aus dem Zähler als auch aus dem Nenner ausgeschlossen.", + "fixproofDefSolvedOverGraded": "Vollständig gelöste Aufgaben von den bisher bewerteten Aufgaben. Ausstehende Aufgaben und ausgeschlossene Läufe stehen nicht im Nenner.", + "fixproofDefRegressions": "Bewertete Läufe, in denen die vorhandene Test-Suite des Pakets nicht mehr bestanden wurde. Ein Strich bedeutet, dass für mindestens einen bewerteten Lauf kein Regressionsergebnis vorliegt.", + "fixproofDefTestEdits": "Änderungen, die der Agent an Testdateien vorgenommen hat. Das Harness setzt sie vor der Bewertung zurück.", + "fixproofDefClaimedOnly": "Läufe, in denen die Zusammenfassung des Agenten Änderungen behauptet hat, die nie auf der Festplatte gelandet sind.", + "fixproofDefMedianMinutes": "Median der tatsächlich verstrichenen Minuten, die der Agent gearbeitet hat, bevor er gestoppt hat oder das Limit von 30 Minuten erreicht war.", + "fixproofDefTrials": "Läufe pro Aufgabe. Ein Versuch ist eine einzelne Stichprobe, kleine Unterschiede sind also Rauschen.", + "fixproofSortAria": "Nach {column} sortieren", + "fixproofDefinitionAria": "Was bedeutet {column}?", + "fixproofChartHeading": "Zeit gegen Index", + "fixproofChartCaption": "Ein Punkt pro Modell und Effort. Die Minuten laufen von langsam links zu schnell rechts, die stärksten Läufe liegen also oben rechts.", + "fixproofChartRegionAria": "Fixproof-Streudiagramm", + "fixproofChartMetricAria": "Metrik des Diagramms", + "fixproofChartLegendAria": "Anbieter", + "fixproofChartAxisMinutes": "Median-Agentminuten pro Aufgabe", + "fixproofChartNote": "schneller + höher ↗", + "fixproofChartPointAria": "{model}, {metric} {value}, {minutes} Median-Minuten" } diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index b66b60e36..348ceec79 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -135,8 +135,8 @@ "changelogAllReleases": "All releases", "changelogLatest": "Latest", "changelogRelease20260612Title": "Agent benchmark, .NET ecosystem, and a 42% lighter install", - "changelogRelease20260612Summary": "This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the homepage, adds .NET as a first-class ecosystem on the new stack graph, and ships a much leaner install. It also fixes four scaffold bugs the benchmark itself uncovered.", - "changelogRelease20260612HighlightBenchmark": "Benchmarked frontier models scaffolding the same project specs three ways: prompt-only, our CLI, and our MCP server. Agents on the MCP path finished up to 7× faster with 4× fewer output tokens; the full results live on the homepage with an interactive chart.", + "changelogRelease20260612Summary": "This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the benchmark page, adds .NET as a first-class ecosystem on the new stack graph, and ships a much leaner install. It also fixes four scaffold bugs the benchmark itself uncovered.", + "changelogRelease20260612HighlightBenchmark": "Benchmarked frontier models scaffolding the same project specs three ways: prompt-only, our CLI, and our MCP server. Agents on the MCP path finished up to 7× faster with 4× fewer output tokens; the full results were published on the benchmark page at the time, with an interactive chart.", "changelogRelease20260612HighlightMcp": "Redesigned the MCP page with one-paste setup for Claude Code, Codex, Gemini CLI, Cursor, VS Code, Claude Desktop, Windsurf, and Zed.", "changelogRelease20260612HighlightDotnet": "Added .NET as a first-class ecosystem, plus an enterprise tier, backend-utils, and Render/Netlify deployment options on the stack graph.", "changelogRelease20260612HighlightInstall": "Cut install size by 42% (122 MB → 71 MB) and the web entry chunk by 32%.", @@ -214,16 +214,15 @@ "mcpWorkflowTitleB": "It builds.", "mcpWorkflowDescription": "Agents use the same schema and compatibility rules as the web builder, so generated commands match what users can configure visually.", "mcpTerminalHeader": "agent session", - "mcpFinalEyebrow": "benchmark-backed", - "mcpFinalTitle": "2.6× faster than", - "mcpFinalTitleEmphasis": "prompt-only.", - "mcpFinalDescription": "In ScaffBench, MCP-guided project creation is faster and more reliable than asking an agent to hand-write a project from scratch.", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "See how agents score on", + "mcpFinalTitleEmphasis": "real issues.", + "mcpFinalDescription": "Fixproof grades coding agents on sealed, real issues from private and public codebases, verified by hidden tests.", "mcpViewBenchmark": "View benchmark", "mcpReadDocs": "Read MCP docs", "mcpStatStructuredTools": "structured tools", "mcpStatReadableResources": "readable resources", "mcpStatConfigurableOptions": "configurable options", - "mcpStatFasterPromptOnly": "faster than prompt-only", "mcpCopyAgentConfiguration": "Copy {agent} configuration", "mcpToolGuidanceDescription": "Workflow rules, field semantics, and critical constraints", "mcpToolSchemaDescription": "Valid options for any category, filterable by ecosystem", @@ -248,34 +247,6 @@ "mcpWorkflowCreateNote": "written to ./my-app", "mcpWorkflowDoneName": "scaffold complete", "mcpWorkflowDoneNote": "run bun install to finish", - "llmBenchmarkDescription": "Measuring coding agents on real fullstack scaffolding tasks - time, tokens, cost, and whether the result actually builds.", - "llmReadBlog": "Read the blog", - "llmTryMcp": "Try out MCP", - "llmBenchmarkMetric": "Benchmark metric", - "llmScatterAria": "Benchmark scatter chart: each point is one model and creation path", - "llmScatterUnmetered": "Not metered on this axis (excluded from the plot): {models}", - "llmFilterModels": "Filter models", - "llmModels": "Models", - "llmClaudeSweep": "Jun 12 sweep", - "llmCodexSweep": "Jun 10 sweep", - "llmLightSweep": "Jun 12 light sweep", - "llmPathCliShort": "BF mention", - "llmPathPromptShort": "Prompt", - "llmPathMcpDetail": "scaffolds through our MCP tools", - "llmPathCliDetail": "agent composes the Better-Fullstack CLI command", - "llmPathPromptDetail": "no Better-Fullstack - agent hand-writes every file", - "llmBuildsPassing": "Builds passing", - "llmAvgScaffoldTime": "Avg scaffold time", - "llmOutputTokens": "Output tokens per scaffold", - "llmFailedBuilds": "Failed builds", - "llmSpeed": "Speed", - "llmTokens": "Tokens", - "llmErrorRate": "Error rate", - "llmMostEfficient": "most efficient ↗", - "llmFastReliable": "fast + reliable ↗", - "llmAgentTitle": "Give your agent the fast path.", - "llmAgentDescription": "One MCP server, every spec-to-scaffold tool the benchmark used. Pick your agent, paste, done.", - "llmAllSupportedClients": "all supported clients", "llmCopyAgentSetupCommand": "Copy {agent} setup command", "llmRunInTerminal": "run in your terminal", "llmPasteInto": "paste into {target}", @@ -487,56 +458,7 @@ "docsSearchSectionsIndexed": "{count} sections indexed", "actionsReset": "Reset", "actionsRandom": "Random", - "runSeoTitle": "Run ScaffBench yourself", - "runSeoDescription": "Reproduce the ScaffBench benchmark locally: clone the harness, point it at any agent CLI or an API key, and score whether the generated projects build.", - "runHeroEyebrow": "Reproduce it", - "runHeroTitleA": "Run ScaffBench", - "runHeroTitleB": "yourself", - "runHeroDescription": "The harness is open source. Clone it, point it at any agent - Claude Code, Codex, opencode, Kilo, or Antigravity for Gemini - and it scaffolds each spec, then scores whether the generated project actually installs and builds. Runs work with a logged-in CLI or a plain API key.", - "runHeroQuickstart": "Quickstart", - "runHeroBrowseReports": "Browse the reports", - "runQuickstartEyebrow": "Quickstart", - "runQuickstartTitle": "Three steps", - "runStepCloneInstall": "Clone & install", - "runStepAuth": "Authenticate your agent", - "runStepRun": "Run the benchmark", - "runLabelClone": "clone the harness", - "runLabelSignin": "sign in to your agent", - "runLabelExportKey": "export a provider key", - "runLabelRunAll": "run all 13 specs, prompt path", - "runLabelTwoPhase": "two-phase", - "runAuthCliTab": "Logged-in CLI", - "runAuthApiTab": "API key", - "runAuthCliDesc": "Use an agent CLI you're already signed into (subscription / OAuth). Log in once, then the harness drives it - no keys in your environment.", - "runAuthApiDesc": "Prefer an API key? Export the provider key and the same agent CLI bills against it - no subscription needed. We publish subscription-driven runs; API runs are untested but supported.", - "runTwoPhaseNote": "Prefer to keep validation clean? Split it into two phases - generate everything first, then validate on its own:", - "runResultsNotePre": "Results - a leaderboard, per-spec pass, wired-libraries, and cost - land in the output directory, in the same shape as the ", - "runResultsNoteLink": "published reports", - "runAgentsEyebrow": "Agents & models", - "runAgentsTitle": "Bring any agent", - "runAgentsDesc": "The provider is inferred from the model id, so one flag picks both the model and the CLI that drives it.", - "runColAgent": "Agent", - "runColModels": "Example models", - "runColAuth": "Auth", - "runFlagsEyebrow": "Flags", - "runFlagsTitle": "Tune the run", - "runFlagModel": "the model to run (see the table above); the provider is inferred from the id", - "runFlagEfforts": "reasoning effort, where the model supports it", - "runFlagPaths": "prompt hand-writes everything; mcp goes through the MCP tools; cli composes the CLI command", - "runFlagSpecs": "the full 13-spec suite by default, or a comma-separated subset of spec ids", - "runFlagPhase": "split the run into a generate phase and a validate phase, validated on its own", - "runFlagOutDir": "where results land; re-use the same directory to resume or validate", - "runCtaEyebrow": "Compare", - "runCtaTitle": "See how your run stacks up", - "runCtaDesc": "Your numbers land in the same format as the leaderboard. Ran something interesting? Open a pull request with your report.", - "runCtaLeaderboard": "View the leaderboard", - "llmRunItYourself": "Run it yourself", "navBenchmark": "Benchmark", - "benchmarkTeaserTitle": "How good are AI models at building your projects?", - "benchmarkTeaserTopModels": "Top models", - "benchmarkTeaserCta": "See the full benchmark", - "benchmarkSeoTitle": "ScaffBench - How good are AI models at building your projects?", - "benchmarkTeaserMcpBody": "The difference is our MCP. Point any coding agent at Better-Fullstack's tools and even a small free model builds almost everything - with a fraction of the tokens and steps.", "navLiveRun": "Live Run", "campaignSeoTitle": "Run Before You Clone | Better Fullstack", "campaignSeoDescription": "Inspect, edit and run a real generated TypeScript project in your browser, then download the ZIP. No signup and no code upload.", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "Full stack", "homeStarterShapeFrontend": "Frontend only", "homeStarterShapeBackend": "Backend only", - "homeStarterShapeMobile": "Mobile app" + "homeStarterShapeMobile": "Mobile app", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof grades coding agents on sealed, real issues from private and public codebases, verified by hidden tests.", + "fixproofSeoTitle": "Fixproof: sealed coding-agent benchmark", + "fixproofClaim": "Real issues from private and public codebases, sealed. Hidden tests decide.", + "fixproofProvenanceSummary": "Every number comes from a recorded unattended run of the named agent CLI on a dedicated Linux bench machine, against the task's base commit and graded by hidden tests that were proven red at that commit and green with the maintainers' fix. Nothing here is hand-scored.", + "fixproofStatusRunLabel": "Dry run", + "fixproofStatusDateLabel": "Date", + "fixproofStatusGradedLabel": "Tasks graded", + "fixproofStatusTrialsLabel": "Trials per task", + "fixproofGradedOfTotal": "{graded} of {total}", + "fixproofBoardHeading": "Board", + "fixproofBoardCaption": "One row per model. Sort by either index. The question mark on a column explains what it counts.", + "fixproofColModel": "Model", + "fixproofColHarness": "Harness", + "fixproofColEffort": "Effort", + "fixproofColResolvedIndex": "Resolved index", + "fixproofColProgressIndex": "Progress index", + "fixproofColSolvedOverGraded": "Solved / graded", + "fixproofColRegressions": "Regressions", + "fixproofColTestEdits": "Test edits reverted", + "fixproofColClaimedOnly": "Claimed, not done", + "fixproofColMedianMinutes": "Median minutes", + "fixproofColRunDate": "Run date", + "fixproofColTrials": "Trials", + "fixproofDefHarness": "The agent CLI that drove the model.", + "fixproofDefResolvedIndex": "Difficulty-weighted share of tasks where every hidden check passed and no regression appeared. This is the headline number.", + "fixproofDefProgressIndex": "Weighted share of each task's requirements that were failing at the base commit and pass after the agent's patch, then difficulty-weighted across tasks. Each requirement carries a weight of 2 (core), 1 or 0.5 (peripheral); requirements already green at base and untested requirements are excluded from both the numerator and denominator.", + "fixproofDefSolvedOverGraded": "Tasks fully resolved out of the tasks graded so far. Pending tasks and excluded runs are not in the denominator.", + "fixproofDefRegressions": "Graded runs where the package's existing test suite stopped passing. A dash means at least one graded run has no regression result.", + "fixproofDefTestEdits": "Edits the agent made to test files. The harness reverts them before grading.", + "fixproofDefClaimedOnly": "Runs where the agent's summary claimed edits that never reached disk.", + "fixproofDefMedianMinutes": "Median wall-clock minutes the agent worked before it stopped or hit the 30 minute cap.", + "fixproofDefTrials": "Runs per task. One trial is a single sample, so read small differences as noise.", + "fixproofSortAria": "Sort by {column}", + "fixproofDefinitionAria": "What does {column} mean?", + "fixproofChartHeading": "Time against index", + "fixproofChartCaption": "One point per model and effort. Minutes run from slow on the left to fast on the right, so the strongest runs sit toward the top right.", + "fixproofChartRegionAria": "Fixproof scatter chart", + "fixproofChartMetricAria": "Chart metric", + "fixproofChartLegendAria": "Vendors", + "fixproofChartAxisMinutes": "Median agent minutes per task", + "fixproofChartNote": "faster + higher ↗", + "fixproofChartPointAria": "{model}, {metric} {value}, {minutes} median minutes" } diff --git a/apps/web/messages/es.json b/apps/web/messages/es.json index 3a772b529..964f0df43 100644 --- a/apps/web/messages/es.json +++ b/apps/web/messages/es.json @@ -132,8 +132,8 @@ "changelogAllReleases": "Todas las versiones", "changelogLatest": "Última", "changelogRelease20260612Title": "Benchmark de agentes, ecosistema .NET e instalación un 42% más ligera", - "changelogRelease20260612Summary": "Esta versión mide cómo los agentes de IA crean scaffolds con Better Fullstack y publica los resultados en la página de inicio, añade .NET como ecosistema de primera clase en el nuevo grafo de stacks y entrega una instalación mucho más ligera. También corrige cuatro errores de scaffold que descubrió el propio benchmark.", - "changelogRelease20260612HighlightBenchmark": "Se probaron modelos de frontera creando los mismos specs de proyecto por tres rutas: solo prompt, nuestra CLI y nuestro servidor MCP. En la ruta MCP, los agentes terminaron hasta 7× más rápido con 4× menos tokens de salida; los resultados completos están en la página de inicio con un gráfico interactivo.", + "changelogRelease20260612Summary": "Esta versión mide cómo los agentes de IA crean scaffolds con Better Fullstack y publica los resultados en la página del benchmark, añade .NET como ecosistema de primera clase en el nuevo grafo de stacks y entrega una instalación mucho más ligera. También corrige cuatro errores de scaffold que descubrió el propio benchmark.", + "changelogRelease20260612HighlightBenchmark": "Se probaron modelos de frontera creando los mismos specs de proyecto por tres rutas: solo prompt, nuestra CLI y nuestro servidor MCP. En la ruta MCP, los agentes terminaron hasta 7× más rápido con 4× menos tokens de salida; los resultados completos se publicaron entonces en la página del benchmark, con un gráfico interactivo.", "changelogRelease20260612HighlightMcp": "Se rediseñó la página MCP con configuración de pegar una vez para Claude Code, Codex, Gemini CLI, Cursor, VS Code, Claude Desktop, Windsurf y Zed.", "changelogRelease20260612HighlightDotnet": "Se añadió .NET como ecosistema de primera clase, además de un nivel enterprise, backend-utils y opciones de despliegue Render/Netlify en el grafo de stacks.", "changelogRelease20260612HighlightInstall": "Se redujo el tamaño de instalación un 42% (122 MB → 71 MB) y el chunk de entrada web un 32%.", @@ -211,16 +211,15 @@ "mcpWorkflowTitleB": "Él construye.", "mcpWorkflowDescription": "Los agentes usan el mismo esquema y reglas de compatibilidad que el constructor web.", "mcpTerminalHeader": "sesión del agente", - "mcpFinalEyebrow": "respaldado por benchmark", - "mcpFinalTitle": "2.6× más rápido que", - "mcpFinalTitleEmphasis": "solo prompt.", - "mcpFinalDescription": "En ScaffBench, crear proyectos guiados por MCP es más rápido y fiable que pedirle a un agente que escriba todo desde cero.", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "Mira cómo puntúan los agentes con", + "mcpFinalTitleEmphasis": "errores reales.", + "mcpFinalDescription": "Fixproof evalúa agentes de programación con errores reales y sellados de bases de código privadas y públicas, verificados por pruebas ocultas.", "mcpViewBenchmark": "Ver benchmark", "mcpReadDocs": "Leer docs MCP", "mcpStatStructuredTools": "herramientas estructuradas", "mcpStatReadableResources": "recursos legibles", "mcpStatConfigurableOptions": "opciones configurables", - "mcpStatFasterPromptOnly": "más rápido que solo prompt", "mcpCopyAgentConfiguration": "Copiar configuración de {agent}", "mcpToolGuidanceDescription": "Reglas de flujo, semántica de campos y restricciones críticas", "mcpToolSchemaDescription": "Opciones válidas para cualquier categoría, filtrables por ecosistema", @@ -245,34 +244,6 @@ "mcpWorkflowCreateNote": "escrito en ./my-app", "mcpWorkflowDoneName": "scaffold completado", "mcpWorkflowDoneNote": "ejecuta bun install para terminar", - "llmBenchmarkDescription": "Mide agentes de programación en tareas reales de scaffolding fullstack: tiempo, tokens, coste y si el resultado realmente compila.", - "llmReadBlog": "Leer el blog", - "llmTryMcp": "Probar MCP", - "llmBenchmarkMetric": "Métrica del benchmark", - "llmScatterAria": "Gráfico de dispersión del benchmark: cada punto es un modelo y una ruta de creación", - "llmScatterUnmetered": "Sin medición en este eje (excluido del gráfico): {models}", - "llmFilterModels": "Filtrar modelos", - "llmModels": "Modelos", - "llmClaudeSweep": "Barrido del 12 jun", - "llmCodexSweep": "Barrido del 10 jun", - "llmLightSweep": "Barrido ligero del 12 jun", - "llmPathCliShort": "Mención BF", - "llmPathPromptShort": "Prompt", - "llmPathMcpDetail": "crea el scaffold mediante nuestras herramientas MCP", - "llmPathCliDetail": "el agente compone el comando CLI de Better-Fullstack", - "llmPathPromptDetail": "sin Better-Fullstack: el agente escribe cada archivo a mano", - "llmBuildsPassing": "Builds que pasan", - "llmAvgScaffoldTime": "Tiempo medio de scaffold", - "llmOutputTokens": "Tokens de salida por scaffold", - "llmFailedBuilds": "Builds fallidos", - "llmSpeed": "Velocidad", - "llmTokens": "Tokens", - "llmErrorRate": "Tasa de error", - "llmMostEfficient": "más eficiente ↗", - "llmFastReliable": "rápido + fiable ↗", - "llmAgentTitle": "Dale a tu agente la ruta rápida.", - "llmAgentDescription": "Un servidor MCP y todas las herramientas para pasar de especificación a scaffold que usó el benchmark. Elige tu agente, pega y listo.", - "llmAllSupportedClients": "todos los clientes soportados", "llmCopyAgentSetupCommand": "Copiar comando de configuración de {agent}", "llmRunInTerminal": "ejecuta en tu terminal", "llmPasteInto": "pega en {target}", @@ -473,56 +444,7 @@ "actionsReset": "Restablecer", "actionsRandom": "Aleatorio", "navAnalytics": "Analítica", - "runSeoTitle": "Ejecuta ScaffBench tú mismo", - "runSeoDescription": "Reproduce localmente el benchmark ScaffBench: clona el harness, apúntalo a cualquier CLI de agente o a una clave API y evalúa si los proyectos generados compilan.", - "runHeroEyebrow": "Reproducirlo", - "runHeroTitleA": "Ejecuta ScaffBench", - "runHeroTitleB": "tú mismo", - "runHeroDescription": "El harness es de código abierto. Clónalo, apúntalo a cualquier agente (Claude Code, Codex, opencode, Kilo o Antigravity para Gemini) y generará la estructura de cada especificación, luego verificará si el proyecto generado realmente se instala y compila. Las ejecuciones funcionan con una CLI con sesión iniciada o con una simple clave API.", - "runHeroQuickstart": "Inicio rápido", - "runHeroBrowseReports": "Consultar los informes", - "runQuickstartEyebrow": "Inicio rápido", - "runQuickstartTitle": "Tres pasos", - "runStepCloneInstall": "Clonar e instalar", - "runStepAuth": "Autentica a tu agente", - "runStepRun": "Ejecutar el benchmark", - "runLabelClone": "clonar el harness", - "runLabelSignin": "iniciar sesión en tu agente", - "runLabelExportKey": "exportar una clave de proveedor", - "runLabelRunAll": "ejecutar las 13 especificaciones, ruta prompt", - "runLabelTwoPhase": "dos fases", - "runAuthCliTab": "CLI con sesión iniciada", - "runAuthApiTab": "clave API", - "runAuthCliDesc": "Usa una CLI de agente en la que ya tengas sesión iniciada (suscripción/OAuth). Inicia sesión una vez y el harness se encarga del resto; sin claves en tu entorno.", - "runAuthApiDesc": "¿Prefieres una clave API? Exporta la clave del proveedor y la misma CLI del agente facturará con ella; no se requiere suscripción. Publicamos ejecuciones con suscripción; las ejecuciones con API no están probadas, pero cuentan con soporte.", - "runTwoPhaseNote": "¿Prefieres mantener la validación limpia? Divídela en dos fases: primero genera todo y luego valida por separado.", - "runResultsNotePre": "Los resultados (una tabla de clasificación, el resultado por especificación, las librerías integradas y el coste) se guardan en el directorio de salida, con el mismo formato que los ", - "runResultsNoteLink": "informes publicados", - "runAgentsEyebrow": "Agentes y modelos", - "runAgentsTitle": "Trae cualquier agente", - "runAgentsDesc": "El proveedor se infiere del id del modelo, así que un solo flag selecciona tanto el modelo como la CLI que lo controla.", - "runColAgent": "Agente", - "runColModels": "Modelos de ejemplo", - "runColAuth": "Autenticación", - "runFlagsEyebrow": "Flags", - "runFlagsTitle": "Ajusta la ejecución", - "runFlagModel": "el modelo a ejecutar (ver la tabla anterior); el proveedor se infiere del id", - "runFlagEfforts": "esfuerzo de razonamiento, cuando el modelo lo admite", - "runFlagPaths": "prompt escribe todo a mano; mcp pasa por las herramientas de MCP; cli compone el comando CLI.", - "runFlagSpecs": "El conjunto completo de 13 especificaciones por defecto, o un subconjunto de identificadores de especificaciones separados por comas.", - "runFlagPhase": "dividir la ejecución en una fase de generación y una fase de validación, validada por sí sola.", - "runFlagOutDir": "donde se almacenan los resultados; reutilizar el mismo directorio para reanudar o validar.", - "runCtaEyebrow": "Comparar", - "runCtaTitle": "Mira cómo se compara tu ejecución", - "runCtaDesc": "Tus resultados se mostrarán en el mismo formato que la tabla de clasificación. ¿Obtuviste algo interesante? Abre un pull request con tu informe.", - "runCtaLeaderboard": "Ver la tabla de clasificación", - "llmRunItYourself": "Ejecútalo tú mismo", "navBenchmark": "Benchmark", - "benchmarkTeaserTitle": "¿Qué tan buenos son los modelos de IA construyendo tus proyectos?", - "benchmarkTeaserTopModels": "Mejores modelos", - "benchmarkTeaserCta": "Ver el benchmark completo", - "benchmarkSeoTitle": "ScaffBench - ¿Qué tan buenos son los modelos de IA construyendo tus proyectos?", - "benchmarkTeaserMcpBody": "La diferencia es nuestro MCP. Conecta cualquier agente de programación a las herramientas de Better-Fullstack y hasta un pequeño modelo gratuito construye casi todo, con una fracción de los tokens y los pasos.", "navUpdates": "Novedades", "navTemplates": "Plantillas", "builderNewFilter": "Nuevo en esta versión", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "Full stack", "homeStarterShapeFrontend": "Solo frontend", "homeStarterShapeBackend": "Solo backend", - "homeStarterShapeMobile": "App móvil" + "homeStarterShapeMobile": "App móvil", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof evalúa agentes de programación con errores reales y sellados de bases de código privadas y públicas, verificados por pruebas ocultas.", + "fixproofSeoTitle": "Fixproof: benchmark sellado para agentes de programación", + "fixproofClaim": "Errores reales de bases de código privadas y públicas, sellados. Deciden las pruebas ocultas.", + "fixproofProvenanceSummary": "Cada cifra sale de una ejecución registrada y sin supervisión de la CLI del agente indicada, en una máquina de pruebas Linux dedicada, contra el commit base de la tarea y puntuada por pruebas ocultas que estaban en rojo en ese commit y en verde con la corrección de los mantenedores. Aquí no hay nada puntuado a mano.", + "fixproofStatusRunLabel": "Ejecución de prueba", + "fixproofStatusDateLabel": "Fecha", + "fixproofStatusGradedLabel": "Tareas evaluadas", + "fixproofStatusTrialsLabel": "Intentos por tarea", + "fixproofGradedOfTotal": "{graded} de {total}", + "fixproofBoardHeading": "Tabla", + "fixproofBoardCaption": "Una fila por modelo. Ordena por cualquiera de los dos índices. El signo de interrogación de cada columna explica qué cuenta.", + "fixproofColModel": "Modelo", + "fixproofColHarness": "Harness", + "fixproofColEffort": "Esfuerzo", + "fixproofColResolvedIndex": "Índice Resolved", + "fixproofColProgressIndex": "Índice Progress", + "fixproofColSolvedOverGraded": "Resueltas / evaluadas", + "fixproofColRegressions": "Regresiones", + "fixproofColTestEdits": "Cambios en pruebas revertidos", + "fixproofColClaimedOnly": "Declarado, no hecho", + "fixproofColMedianMinutes": "Mediana de minutos", + "fixproofColRunDate": "Fecha de ejecución", + "fixproofColTrials": "Intentos", + "fixproofDefHarness": "La CLI de agente que condujo el modelo.", + "fixproofDefResolvedIndex": "Proporción ponderada por dificultad de las tareas en las que pasaron todas las comprobaciones ocultas y no apareció ninguna regresión. Es la cifra principal.", + "fixproofDefProgressIndex": "Proporción ponderada de los requisitos de cada tarea que fallaban en el commit base y pasan tras el parche del agente, ponderada después por dificultad entre tareas. Cada requisito lleva un peso de 2 (central), 1 o 0,5 (periférico); los requisitos que ya pasaban en el commit base y los requisitos no probados se excluyen tanto del numerador como del denominador.", + "fixproofDefSolvedOverGraded": "Tareas resueltas por completo sobre las tareas evaluadas hasta ahora. Las tareas pendientes y las ejecuciones excluidas no están en el denominador.", + "fixproofDefRegressions": "Ejecuciones evaluadas en las que la suite de pruebas existente del paquete dejó de pasar. Un guion indica que al menos una ejecución evaluada no tiene un resultado de regresiones.", + "fixproofDefTestEdits": "Cambios que el agente hizo en archivos de prueba. El harness los revierte antes de evaluar.", + "fixproofDefClaimedOnly": "Ejecuciones en las que el resumen del agente declaró cambios que nunca llegaron al disco.", + "fixproofDefMedianMinutes": "Mediana de minutos reales que el agente trabajó antes de parar o de llegar al límite de 30 minutos.", + "fixproofDefTrials": "Ejecuciones por tarea. Un intento es una sola muestra, así que lee las diferencias pequeñas como ruido.", + "fixproofSortAria": "Ordenar por {column}", + "fixproofDefinitionAria": "¿Qué significa {column}?", + "fixproofChartHeading": "Tiempo frente al índice", + "fixproofChartCaption": "Un punto por modelo y esfuerzo. Los minutos van de lento a la izquierda a rápido a la derecha, así que las mejores ejecuciones quedan arriba a la derecha.", + "fixproofChartRegionAria": "Gráfico de dispersión de Fixproof", + "fixproofChartMetricAria": "Métrica del gráfico", + "fixproofChartLegendAria": "Proveedores", + "fixproofChartAxisMinutes": "Mediana de minutos del agente por tarea", + "fixproofChartNote": "más rápido + más alto ↗", + "fixproofChartPointAria": "{model}, {metric} {value}, {minutes} minutos de mediana" } diff --git a/apps/web/messages/fr.json b/apps/web/messages/fr.json index 1449c25da..fd797f35e 100644 --- a/apps/web/messages/fr.json +++ b/apps/web/messages/fr.json @@ -132,8 +132,8 @@ "changelogAllReleases": "Toutes les versions", "changelogLatest": "Dernier", "changelogRelease20260612Title": "Benchmark d'agent, écosystème .NET et installation 42 % plus légère", - "changelogRelease20260612Summary": "Cette version évalue la façon dont les agents AI échafaudent avec Better Fullstack et publie les résultats sur la page d'accueil, ajoute .NET en tant qu'écosystème de première classe sur le nouveau graphique de pile et fournit une installation beaucoup plus légère. Elle corrige également quatre bugs d’échafaudage découverts par le benchmark lui-même.", - "changelogRelease20260612HighlightBenchmark": "Nous avons comparé des modèles de pointe qui échafaudent les mêmes spécifications de projet de trois manières : par invite uniquement, par notre CLI et par notre serveur MCP. Les agents sur le chemin MCP ont terminé jusqu'à 7 fois plus vite avec 4 fois moins de jetons de sortie ; les résultats complets sont disponibles sur la page d'accueil avec un graphique interactif.", + "changelogRelease20260612Summary": "Cette version évalue la façon dont les agents AI échafaudent avec Better Fullstack et publie les résultats sur la page du benchmark, ajoute .NET en tant qu'écosystème de première classe sur le nouveau graphique de pile et fournit une installation beaucoup plus légère. Elle corrige également quatre bugs d’échafaudage découverts par le benchmark lui-même.", + "changelogRelease20260612HighlightBenchmark": "Nous avons comparé des modèles de pointe qui échafaudent les mêmes spécifications de projet de trois manières : par invite uniquement, par notre CLI et par notre serveur MCP. Les agents sur le chemin MCP ont terminé jusqu'à 7 fois plus vite avec 4 fois moins de jetons de sortie ; les résultats complets ont été publiés à l'époque sur la page du benchmark, avec un graphique interactif.", "changelogRelease20260612HighlightMcp": "Refonte de la page MCP avec une configuration en un seul collage pour Claude Code, Codex, Gemini CLI, Cursor, VS Code, Claude Desktop, Windsurf et Zed.", "changelogRelease20260612HighlightDotnet": "Ajout de .NET en tant qu'écosystème de première classe, ainsi que d'un niveau entreprise, d'utilitaires backend et d'options de déploiement Render/Netlify sur le graphique de pile.", "changelogRelease20260612HighlightInstall": "Réduisez la taille de l'installation de 42 % (122 Mo → 71 Mo) et la taille de l'entrée Web de 32 %.", @@ -211,16 +211,15 @@ "mcpWorkflowTitleB": "Cela construit.", "mcpWorkflowDescription": "Les agents utilisent le même schéma et les mêmes règles de compatibilité que le générateur Web. Les commandes générées correspondent donc à ce que les utilisateurs peuvent configurer visuellement.", "mcpTerminalHeader": "session d'agent", - "mcpFinalEyebrow": "appuyé par un benchmark", - "mcpFinalTitle": "2,6 fois plus rapide que", - "mcpFinalTitleEmphasis": "invite uniquement.", - "mcpFinalDescription": "Dans ScaffBench, la création de projet guidée par MCP est plus rapide et plus fiable que de demander à un agent d'écrire manuellement un projet à partir de zéro.", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "Voyez comment les agents s'en sortent sur", + "mcpFinalTitleEmphasis": "des bugs réels.", + "mcpFinalDescription": "Fixproof évalue les agents de codage sur des bugs réels et scellés issus de bases de code privées et publiques, vérifiés par des tests cachés.", "mcpViewBenchmark": "Voir le benchmark", "mcpReadDocs": "Lire la documentation MCP", "mcpStatStructuredTools": "outils structurés", "mcpStatReadableResources": "ressources lisibles", "mcpStatConfigurableOptions": "options configurables", - "mcpStatFasterPromptOnly": "plus rapide que l'invite uniquement", "mcpCopyAgentConfiguration": "Copier la configuration {agent}", "mcpToolGuidanceDescription": "Règles de workflow, sémantique des champs et contraintes critiques", "mcpToolSchemaDescription": "Options valides pour n'importe quelle catégorie, filtrables par écosystème", @@ -245,34 +244,6 @@ "mcpWorkflowCreateNote": "écrit dans ./my-app", "mcpWorkflowDoneName": "échafaudage terminé", "mcpWorkflowDoneNote": "lancez bun install pour terminer", - "llmBenchmarkDescription": "Mesurer les agents de codage sur de véritables tâches d'échafaudage fullstack : temps, jetons, coût et si le résultat se compile réellement.", - "llmReadBlog": "Lire le blog", - "llmTryMcp": "Essayez MCP", - "llmBenchmarkMetric": "Métrique de référence", - "llmScatterAria": "Diagramme de dispersion de référence : chaque point correspond à un modèle et à un chemin de création", - "llmScatterUnmetered": "Non mesuré sur cet axe (exclu du graphique) : {models}", - "llmFilterModels": "Filtrer les modèles", - "llmModels": "Modèles", - "llmClaudeSweep": "Balayage du 12 juin", - "llmCodexSweep": "Balayage du 10 juin", - "llmLightSweep": "Balayage léger du 12 juin", - "llmPathCliShort": "Mention BF", - "llmPathPromptShort": "Invite", - "llmPathMcpDetail": "échafaudages grâce à nos outils MCP", - "llmPathCliDetail": "l'agent compose la commande Better-Fullstack CLI", - "llmPathPromptDetail": "pas de Better-Fullstack - l'agent écrit manuellement chaque fichier", - "llmBuildsPassing": "Constructions réussies", - "llmAvgScaffoldTime": "Temps moyen d'échafaudage", - "llmOutputTokens": "Jetons de sortie par échafaudage", - "llmFailedBuilds": "Constructions échouées", - "llmSpeed": "Vitesse", - "llmTokens": "Jetons", - "llmErrorRate": "Taux d'erreur", - "llmMostEfficient": "le plus efficace ↗", - "llmFastReliable": "rapide + fiable ↗", - "llmAgentTitle": "Donnez à votre agent la voie rapide.", - "llmAgentDescription": "Un serveur MCP, chaque outil de spécification à échafaudage utilisé par le benchmark. Choisissez votre agent, collez, c'est fait.", - "llmAllSupportedClients": "tous les clients pris en charge", "llmCopyAgentSetupCommand": "Copier la commande de configuration {agent}", "llmRunInTerminal": "exécuter dans votre terminal", "llmPasteInto": "collez dans {target}", @@ -473,56 +444,7 @@ "actionsReset": "Réinitialiser", "actionsRandom": "Aléatoire", "navAnalytics": "Analytique", - "runSeoTitle": "Exécutez ScaffBench vous-même", - "runSeoDescription": "Reproduisez localement le benchmark ScaffBench : clonez le framework, pointez-le vers n’importe quelle interface de ligne de commande d’agent ou une clé API, et vérifiez si les projets générés sont compilés.", - "runHeroEyebrow": "Reproduisez-le", - "runHeroTitleA": "Exécuter ScaffBench", - "runHeroTitleB": "vous-même", - "runHeroDescription": "Ce framework est open source. Clonez-le, configurez-le avec n'importe quel agent (Claude Code, Codex, opencode, Kilo ou Antigravity pour Gemini) et il générera la structure de chaque spécification, puis vérifiera si le projet généré s'installe et se compile correctement. Il fonctionne avec une interface de ligne de commande (CLI) ou une simple clé API.", - "runHeroQuickstart": "Démarrage rapide", - "runHeroBrowseReports": "Consultez les rapports", - "runQuickstartEyebrow": "Démarrage rapide", - "runQuickstartTitle": "Trois étapes", - "runStepCloneInstall": "Cloner et installer", - "runStepAuth": "Authentifiez votre agent", - "runStepRun": "Exécuter le test de performance", - "runLabelClone": "cloner le harnais", - "runLabelSignin": "Connectez-vous à votre agent", - "runLabelExportKey": "exporter une clé de fournisseur", - "runLabelRunAll": "exécuter les 13 spécifications, chemin d'invite", - "runLabelTwoPhase": "en deux phases", - "runAuthCliTab": "Interface de ligne de commande (CLI) connectée", - "runAuthApiTab": "Clé API", - "runAuthCliDesc": "Utilisez l'interface de ligne de commande d'un agent auquel vous êtes déjà connecté (abonnement/OAuth). Connectez-vous une seule fois, puis le système prend le relais ; aucune clé n'est requise dans votre environnement.", - "runAuthApiDesc": "Vous préférez une clé API ? Exportez la clé du fournisseur : la même interface de ligne de commande de l’agent facturera automatiquement l’exécution avec cette clé, sans abonnement. Nous publions les exécutions sur abonnement ; les exécutions via API ne sont pas testées, mais sont prises en charge.", - "runTwoPhaseNote": "Vous préférez une validation simple ? Divisez-la en deux phases : générez d’abord tout, puis validez séparément :", - "runResultsNotePre": "Les résultats - un classement, un passage par spécification, les bibliothèques câblées et le coût - sont enregistrés dans le répertoire de sortie, sous la même forme que le ", - "runResultsNoteLink": "rapports publiés", - "runAgentsEyebrow": "Agents et modèles", - "runAgentsTitle": "Amenez n'importe quel agent", - "runAgentsDesc": "Le fournisseur est déduit de l'identifiant du modèle ; un seul indicateur sélectionne donc à la fois le modèle et l'interface de ligne de commande qui le pilote.", - "runColAgent": "Agent", - "runColModels": "Exemples de modèles", - "runColAuth": "Authentification", - "runFlagsEyebrow": "Indicateurs", - "runFlagsTitle": "Ajustez l'exécution", - "runFlagModel": "le modèle à exécuter (voir le tableau ci-dessus) ; le fournisseur est déduit de l’identifiant", - "runFlagEfforts": "effort de raisonnement, lorsque le modèle le prend en charge", - "runFlagPaths": "prompt saisit tout manuellement ; mcp parcourt les outils MCP ; cli compose la commande CLI", - "runFlagSpecs": "par défaut, la suite complète de 13 spécifications, ou un sous-ensemble d'identifiants de spécifications séparés par des virgules.", - "runFlagPhase": "diviser l'exécution en une phase de génération et une phase de validation, validée séparément.", - "runFlagOutDir": "où les résultats sont enregistrés ; réutiliser le même répertoire pour reprendre ou valider", - "runCtaEyebrow": "Comparer", - "runCtaTitle": "Voyez comment votre exécution se compare.", - "runCtaDesc": "Vos résultats s'affichent au même format que le classement. Vous avez réalisé une expérience intéressante ? Soumettez une pull request avec votre rapport.", - "runCtaLeaderboard": "Consultez le classement", - "llmRunItYourself": "Exécutez-le vous-même", "navBenchmark": "Benchmark", - "benchmarkTeaserTitle": "Les modèles d'IA sont-ils vraiment bons pour créer vos projets ?", - "benchmarkTeaserTopModels": "Meilleurs modèles", - "benchmarkTeaserCta": "Voir le benchmark complet", - "benchmarkSeoTitle": "ScaffBench - Les modèles d'IA sont-ils vraiment bons pour créer vos projets ?", - "benchmarkTeaserMcpBody": "La différence, c'est notre MCP. Connectez n'importe quel agent de codage aux outils de Better-Fullstack et même un petit modèle gratuit construit presque tout, avec une fraction des jetons et des étapes.", "navUpdates": "Nouveautés", "navTemplates": "Modèles", "builderNewFilter": "Nouveau dans cette version", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "Full stack", "homeStarterShapeFrontend": "Frontend uniquement", "homeStarterShapeBackend": "Backend uniquement", - "homeStarterShapeMobile": "Application mobile" + "homeStarterShapeMobile": "Application mobile", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof évalue les agents de codage sur des bugs réels et scellés issus de bases de code privées et publiques, vérifiés par des tests cachés.", + "fixproofSeoTitle": "Fixproof : benchmark scellé pour agents de codage", + "fixproofClaim": "Des bugs réels issus de bases de code privées et publiques, scellés. Ce sont les tests cachés qui tranchent.", + "fixproofProvenanceSummary": "Chaque chiffre sort d'une exécution enregistrée et sans supervision du CLI d'agent indiqué, sur une machine de test Linux dédiée, face au commit de base de la tâche et noté par des tests cachés vérifiés rouges à ce commit et verts avec le correctif des mainteneurs. Rien ici n'est noté à la main.", + "fixproofStatusRunLabel": "Essai à blanc", + "fixproofStatusDateLabel": "Date", + "fixproofStatusGradedLabel": "Tâches évaluées", + "fixproofStatusTrialsLabel": "Essais par tâche", + "fixproofGradedOfTotal": "{graded} sur {total}", + "fixproofBoardHeading": "Tableau", + "fixproofBoardCaption": "Une ligne par modèle. Triez selon l'un ou l'autre indice. Le point d'interrogation d'une colonne explique ce qu'elle compte.", + "fixproofColModel": "Modèle", + "fixproofColHarness": "Harness", + "fixproofColEffort": "Effort", + "fixproofColResolvedIndex": "Indice Resolved", + "fixproofColProgressIndex": "Indice Progress", + "fixproofColSolvedOverGraded": "Résolues / évaluées", + "fixproofColRegressions": "Régressions", + "fixproofColTestEdits": "Modifications de tests annulées", + "fixproofColClaimedOnly": "Annoncé, pas fait", + "fixproofColMedianMinutes": "Minutes médianes", + "fixproofColRunDate": "Date d'exécution", + "fixproofColTrials": "Essais", + "fixproofDefHarness": "La CLI d'agent qui a piloté le modèle.", + "fixproofDefResolvedIndex": "Part des tâches, pondérée par la difficulté, où toutes les vérifications cachées sont passées et où aucune régression n'est apparue. C'est le chiffre principal.", + "fixproofDefProgressIndex": "Part pondérée des exigences de chaque tâche qui échouaient au commit de base et passent après le correctif de l'agent, puis pondérée par la difficulté sur l'ensemble des tâches. Chaque exigence porte un poids de 2 (cœur), 1 ou 0,5 (périphérique) ; les exigences déjà satisfaites au commit de base et les exigences non testées sont exclues du numérateur comme du dénominateur.", + "fixproofDefSolvedOverGraded": "Tâches entièrement résolues sur les tâches évaluées jusqu'ici. Les tâches en attente et les exécutions exclues ne sont pas au dénominateur.", + "fixproofDefRegressions": "Exécutions évaluées où la suite de tests existante du paquet a cessé de passer. Un tiret indique qu’au moins une exécution évaluée n’a pas de résultat de régression.", + "fixproofDefTestEdits": "Modifications que l'agent a apportées aux fichiers de test. Le harness les annule avant l'évaluation.", + "fixproofDefClaimedOnly": "Exécutions où le résumé de l'agent annonçait des modifications qui n'ont jamais atteint le disque.", + "fixproofDefMedianMinutes": "Médiane des minutes réelles travaillées par l'agent avant qu'il s'arrête ou atteigne la limite de 30 minutes.", + "fixproofDefTrials": "Exécutions par tâche. Un essai est un échantillon unique : lisez les petits écarts comme du bruit.", + "fixproofSortAria": "Trier par {column}", + "fixproofDefinitionAria": "Que signifie {column} ?", + "fixproofChartHeading": "Temps et indice", + "fixproofChartCaption": "Un point par modèle et par effort. Les minutes vont de lent à gauche à rapide à droite, donc les meilleures exécutions se placent en haut à droite.", + "fixproofChartRegionAria": "Nuage de points Fixproof", + "fixproofChartMetricAria": "Métrique du graphique", + "fixproofChartLegendAria": "Fournisseurs", + "fixproofChartAxisMinutes": "Minutes médianes de l'agent par tâche", + "fixproofChartNote": "plus rapide + plus haut ↗", + "fixproofChartPointAria": "{model}, {metric} {value}, {minutes} minutes médianes" } diff --git a/apps/web/messages/ja.json b/apps/web/messages/ja.json index 8238ca9bb..af879daf8 100644 --- a/apps/web/messages/ja.json +++ b/apps/web/messages/ja.json @@ -132,8 +132,8 @@ "changelogAllReleases": "すべてのリリース", "changelogLatest": "最新", "changelogRelease20260612Title": "エージェントのベンチマーク、.NET エコシステム、および 42% 軽量のインストール", - "changelogRelease20260612Summary": "このリリースでは、AI エージェントが Better Fullstack でどのようにスキャフォールディングするかをベンチマークし、その結果をホームページで公開し、新しいスタック グラフにファーストクラスのエコシステムとして .NET を追加し、より無駄のないインストールをリリースします。また、ベンチマーク自体が発見した 4 つのスキャフォールドのバグも修正されています。", - "changelogRelease20260612HighlightBenchmark": "ベンチマークされたフロンティア モデルは、同じプロジェクト仕様を 3 つの方法 (プロンプトのみ、CLI、MCP サーバー) でスキャフォールディングします。MCP パス上のエージェントは、出力トークンが 4 分の 1 で、最大 7 倍の速さで完了しました。完全な結果は、インタラクティブなグラフとともにホームページに表示されます。", + "changelogRelease20260612Summary": "このリリースでは、AI エージェントが Better Fullstack でどのようにスキャフォールディングするかをベンチマークし、その結果をベンチマークページで公開し、新しいスタック グラフにファーストクラスのエコシステムとして .NET を追加し、より無駄のないインストールをリリースします。また、ベンチマーク自体が発見した 4 つのスキャフォールドのバグも修正されています。", + "changelogRelease20260612HighlightBenchmark": "ベンチマークされたフロンティア モデルは、同じプロジェクト仕様を 3 つの方法 (プロンプトのみ、CLI、MCP サーバー) でスキャフォールディングします。MCP パス上のエージェントは、出力トークンが 4 分の 1 で、最大 7 倍の速さで完了しました。完全な結果は当時、インタラクティブなグラフとともにベンチマークページで公開されました。", "changelogRelease20260612HighlightMcp": "Claude Code、Codex、Gemini CLI、Cursor、VS Code、Claude Desktop、Windsurf、および Zed の 1 回の貼り付け設定で MCP ページを再設計しました。", "changelogRelease20260612HighlightDotnet": "ファーストクラスのエコシステムとして .NET を追加し、さらにエンタープライズ層、バックエンド ユーティリティ、およびスタック グラフ上の Render/Netlify デプロイメント オプションを追加しました。", "changelogRelease20260612HighlightInstall": "インストール サイズが 42% (122 MB → 71 MB)、Web エントリ チャンクが 32% 削減されました。", @@ -211,16 +211,15 @@ "mcpWorkflowTitleB": "それは構築されます。", "mcpWorkflowDescription": "エージェントは Web ビルダーと同じスキーマと互換性ルールを使用するため、生成されたコマンドはユーザーが視覚的に設定できるものと一致します。", "mcpTerminalHeader": "エージェントセッション", - "mcpFinalEyebrow": "ベンチマークに裏付けられた", - "mcpFinalTitle": "プロンプトのみより", - "mcpFinalTitleEmphasis": "2.6倍高速。", - "mcpFinalDescription": "ScaffBench では、MCP のガイド付きプロジェクト作成は、エージェントにプロジェクトを最初から手書きで作成させるよりも速く、信頼性が高くなります。", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "エージェントの成績が分かるのは", + "mcpFinalTitleEmphasis": "実際の不具合。", + "mcpFinalDescription": "Fixproof は、非公開および公開コードベースから集めた封印済みの実際の不具合でコーディングエージェントを採点し、非公開テストで検証します。", "mcpViewBenchmark": "ベンチマークを表示する", "mcpReadDocs": "MCP ドキュメントを読む", "mcpStatStructuredTools": "構造化されたツール", "mcpStatReadableResources": "読み取り可能なリソース", "mcpStatConfigurableOptions": "構成可能なオプション", - "mcpStatFasterPromptOnly": "プロンプトのみよりも高速", "mcpCopyAgentConfiguration": "{agent} 構成をコピーします", "mcpToolGuidanceDescription": "ワークフロー ルール、フィールド セマンティクス、および重要な制約", "mcpToolSchemaDescription": "あらゆるカテゴリに有効なオプション、エコシステムごとにフィルタリング可能", @@ -245,34 +244,6 @@ "mcpWorkflowCreateNote": "./my-app に書き込まれます", "mcpWorkflowDoneName": "スキャフォールド完了", "mcpWorkflowDoneNote": "bun install を実行して完了します", - "llmBenchmarkDescription": "実際のフルスタック スキャフォールディング タスクにおけるコーディング エージェントを測定します (時間、トークン、コスト、結果が実際にビルドされるかどうか)。", - "llmReadBlog": "ブログを読む", - "llmTryMcp": "MCP を試してみる", - "llmBenchmarkMetric": "ベンチマーク指標", - "llmScatterAria": "ベンチマーク散布図: 各ポイントは 1 つのモデルと作成パスです", - "llmScatterUnmetered": "この軸では計測されていません(プロットから除外):{models}", - "llmFilterModels": "フィルターモデル", - "llmModels": "モデル", - "llmClaudeSweep": "6月12日のスイープ", - "llmCodexSweep": "6月10日のスイープ", - "llmLightSweep": "6月12日 ライトスイープ", - "llmPathCliShort": "BFの言及", - "llmPathPromptShort": "プロンプト", - "llmPathMcpDetail": "MCP ツールを使用した足場", - "llmPathCliDetail": "エージェントは Better-Fullstack CLI コマンドを作成します", - "llmPathPromptDetail": "Better-Fullstack なし - エージェントがすべてのファイルを手書きします", - "llmBuildsPassing": "ビルドの合格", - "llmAvgScaffoldTime": "平均スキャフォールド時間", - "llmOutputTokens": "スキャフォールドごとの出力トークン", - "llmFailedBuilds": "失敗したビルド", - "llmSpeed": "スピード", - "llmTokens": "トークン", - "llmErrorRate": "エラー率", - "llmMostEfficient": "最も効率的 ↗", - "llmFastReliable": "高速 + 信頼性の高い ↗", - "llmAgentTitle": "エージェントに高速パスを提供します。", - "llmAgentDescription": "1 台の MCP サーバー、ベンチマークで使用されたすべての仕様から足場へのツール。エージェントを選択し、貼り付けて完了です。", - "llmAllSupportedClients": "サポートされているすべてのクライアント", "llmCopyAgentSetupCommand": "{agent} セットアップ コマンドをコピーします", "llmRunInTerminal": "ターミナルで実行します", "llmPasteInto": "{target} に貼り付け", @@ -473,56 +444,7 @@ "actionsReset": "リセット", "actionsRandom": "ランダム", "navAnalytics": "分析", - "runSeoTitle": "ScaffBenchを自分で実行してみましょう", - "runSeoDescription": "ScaffBenchベンチマークをローカルで再現するには、ハーネスをクローンし、任意のエージェントCLIまたはAPIキーを指定して、生成されたプロジェクトがビルドされるかどうかをスコアリングします。", - "runHeroEyebrow": "それを再現する", - "runHeroTitleA": "ScaffBenchを実行する", - "runHeroTitleB": "あなた自身", - "runHeroDescription": "このハーネスはオープンソースです。クローンを作成し、Claude Code、Codex、opencode、Kilo、またはGemini用のAntigravityといった任意のエージェントを指定すると、各仕様のひな形が生成され、生成されたプロジェクトが実際にインストールおよびビルドできるかどうかが評価されます。ログイン済みのCLIまたは通常のAPIキーを使用して作業を実行できます。", - "runHeroQuickstart": "クイックスタート", - "runHeroBrowseReports": "レポートを閲覧する", - "runQuickstartEyebrow": "クイックスタート", - "runQuickstartTitle": "3つのステップ", - "runStepCloneInstall": "クローンしてインストール", - "runStepAuth": "エージェントを認証する", - "runStepRun": "ベンチマークを実行する", - "runLabelClone": "ハーネスをクローンする", - "runLabelSignin": "エージェントにサインイン", - "runLabelExportKey": "プロバイダーキーをエクスポートする", - "runLabelRunAll": "全13仕様を実行(プロンプトパス)", - "runLabelTwoPhase": "2フェーズ", - "runAuthCliTab": "ログイン済みCLI", - "runAuthApiTab": "APIキー", - "runAuthCliDesc": "既にサインイン済みのエージェントCLI(サブスクリプション/OAuth)を使用してください。一度ログインすれば、あとはハーネスが自動的に操作します。環境内にキーは不要です。", - "runAuthApiDesc": "APIキーをご希望ですか?プロバイダーキーをエクスポートすれば、同じエージェントCLIがそのキーに基づいて課金します。サブスクリプションは不要です。弊社ではサブスクリプションベースの実行を公開していますが、API実行はテストされていませんがサポート対象です。", - "runTwoPhaseNote": "検証を簡潔に保ちたい場合は、2つのフェーズに分割します。まずすべてを生成し、次に検証を単独で行います。", - "runResultsNotePre": "結果(リーダーボード、仕様ごとの合否、wired-libraries、コスト)は、出力ディレクトリに保存されます。形式は次と同じです: ", - "runResultsNoteLink": "公表された報告書", - "runAgentsEyebrow": "エージェントとモデル", - "runAgentsTitle": "どんなエージェントでも", - "runAgentsDesc": "プロバイダーはモデルIDから推測されるため、1つのフラグでモデルと、それを駆動するCLIの両方を選択できます。", - "runColAgent": "エージェント", - "runColModels": "サンプルモデル", - "runColAuth": "認証", - "runFlagsEyebrow": "フラグ", - "runFlagsTitle": "実行の調整", - "runFlagModel": "実行するモデル(上記の表を参照)。プロバイダーはIDから推測されます。", - "runFlagEfforts": "モデルがそれをサポートする推論努力", - "runFlagPaths": "prompt はすべてを手書きで入力し、mcp は MCP ツールを経由し、cli は CLI コマンドを構成します。", - "runFlagSpecs": "デフォルトでは13個の仕様すべて、またはカンマ区切りの仕様IDのサブセット", - "runFlagPhase": "実行を生成フェーズと検証フェーズに分割し、それぞれを個別に検証する", - "runFlagOutDir": "結果が保存される場所。同じディレクトリを再利用して再開または検証します。", - "runCtaEyebrow": "比較する", - "runCtaTitle": "あなたの実行結果がどれだけ通用するか見てみましょう", - "runCtaDesc": "あなたの数値はリーダーボードと同じ形式で表示されます。何か興味深い結果が出ましたか?レポートを添えてプルリクエストを作成してください。", - "runCtaLeaderboard": "リーダーボードを見る", - "llmRunItYourself": "自分で実行してください", "navBenchmark": "ベンチマーク", - "benchmarkTeaserTitle": "AIモデルはあなたのプロジェクトをどれだけうまく構築できるか?", - "benchmarkTeaserTopModels": "トップモデル", - "benchmarkTeaserCta": "ベンチマーク全体を見る", - "benchmarkSeoTitle": "ScaffBench - AIモデルはあなたのプロジェクトをどれだけうまく構築できるか?", - "benchmarkTeaserMcpBody": "違いは私たちのMCPです。あらゆるコーディングエージェントをBetter-Fullstackのツールに向ければ、小さな無料モデルでもわずかなトークンとステップでほぼすべてを構築します。", "navUpdates": "アップデート", "navTemplates": "テンプレート", "builderNewFilter": "このリリースの新機能", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "フルスタック", "homeStarterShapeFrontend": "フロントエンドのみ", "homeStarterShapeBackend": "バックエンドのみ", - "homeStarterShapeMobile": "モバイルアプリ" + "homeStarterShapeMobile": "モバイルアプリ", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof は、非公開および公開コードベースから集めた封印済みの実際の不具合でコーディングエージェントを採点し、非公開テストで検証します。", + "fixproofSeoTitle": "Fixproof: 封印されたコーディングエージェントのベンチマーク", + "fixproofClaim": "非公開および公開コードベースの実際の不具合を封印。判定するのは非公開テストです。", + "fixproofProvenanceSummary": "すべての数値は、専用の Linux ベンチマシンで指定のエージェント CLI を無人実行した記録から得ています。判定はタスクのベースコミットに対して行い、そのコミットでは赤、メンテナの修正では緑になることを確認済みの非公開テストが採点します。手作業での採点はありません。", + "fixproofStatusRunLabel": "ドライラン", + "fixproofStatusDateLabel": "日付", + "fixproofStatusGradedLabel": "採点済みタスク", + "fixproofStatusTrialsLabel": "タスクあたりの試行数", + "fixproofGradedOfTotal": "{total} 件中 {graded} 件", + "fixproofBoardHeading": "ボード", + "fixproofBoardCaption": "1 行が 1 モデルです。どちらの指数でも並べ替えられます。列の疑問符は、その列が何を数えているかを説明します。", + "fixproofColModel": "モデル", + "fixproofColHarness": "ハーネス", + "fixproofColEffort": "推論強度", + "fixproofColResolvedIndex": "Resolved 指数", + "fixproofColProgressIndex": "Progress 指数", + "fixproofColSolvedOverGraded": "解決 / 採点", + "fixproofColRegressions": "リグレッション", + "fixproofColTestEdits": "差し戻したテスト変更", + "fixproofColClaimedOnly": "主張のみ、未実施", + "fixproofColMedianMinutes": "所要時間の中央値 (分)", + "fixproofColRunDate": "実行日", + "fixproofColTrials": "試行数", + "fixproofDefHarness": "モデルを動かしたエージェント CLI です。", + "fixproofDefResolvedIndex": "すべての非公開チェックに合格し、リグレッションも出なかったタスクの割合を、難易度で重み付けした値です。これが中心となる数値です。", + "fixproofDefProgressIndex": "各タスクの要件のうち、ベースコミットでは失敗しエージェントのパッチ後に成功するものの重み付き割合を求め、さらにタスク間で難易度による重み付けを行った値です。各要件の重みは 2 (中核)、1 または 0.5 (周辺) です。ベースコミットの時点ですでに成功していた要件と未テストの要件は、分子と分母の両方から除外します。", + "fixproofDefSolvedOverGraded": "これまでに採点したタスクのうち、完全に解決したタスクの数です。保留中のタスクと除外した実行は分母に含みません。", + "fixproofDefRegressions": "パッケージ既存のテストスイートが通らなくなった採点済みの実行です。 ダッシュは、採点済みの実行のうち少なくとも1件でリグレッションの結果が不明であることを示します。", + "fixproofDefTestEdits": "エージェントがテストファイルに加えた変更です。ハーネスが採点前に差し戻します。", + "fixproofDefClaimedOnly": "エージェントの要約が、ディスクに届かなかった変更を主張した実行です。", + "fixproofDefMedianMinutes": "エージェントが停止するか 30 分の上限に達するまでに作業した実時間の中央値 (分) です。", + "fixproofDefTrials": "タスクあたりの実行回数です。1 回の試行はサンプル 1 つなので、小さな差はノイズとして読んでください。", + "fixproofSortAria": "{column} で並べ替え", + "fixproofDefinitionAria": "{column} の意味は?", + "fixproofChartHeading": "所要時間と指数", + "fixproofChartCaption": "モデルと推論強度の組み合わせごとに 1 点です。横軸は左が遅く右が速いので、優れた実行ほど右上に寄ります。", + "fixproofChartRegionAria": "Fixproof 散布図", + "fixproofChartMetricAria": "グラフの指標", + "fixproofChartLegendAria": "ベンダー", + "fixproofChartAxisMinutes": "タスクあたりのエージェント所要時間の中央値 (分)", + "fixproofChartNote": "速い + 高い ↗", + "fixproofChartPointAria": "{model}、{metric} {value}、中央値 {minutes} 分" } diff --git a/apps/web/messages/ko.json b/apps/web/messages/ko.json index 65fb63c27..dfc11a896 100644 --- a/apps/web/messages/ko.json +++ b/apps/web/messages/ko.json @@ -132,8 +132,8 @@ "changelogAllReleases": "모든 릴리스", "changelogLatest": "최신", "changelogRelease20260612Title": "에이전트 벤치마크, .NET 생태계 및 42% 더 가벼운 설치", - "changelogRelease20260612Summary": "이 릴리스에서는 AI 에이전트가 Better Fullstack을 사용하여 스캐폴드하고 결과를 홈페이지에 게시하는 방법을 벤치마킹하고, 새 스택 그래프에 .NET을 일류 생태계로 추가하고, 훨씬 더 간결한 설치를 제공합니다. 또한 벤치마크 자체에서 발견한 4가지 스캐폴드 버그도 수정합니다.", - "changelogRelease20260612HighlightBenchmark": "동일한 프로젝트 사양을 세 가지 방식으로 스캐폴딩하는 벤치마킹된 프론티어 모델: 프롬프트 전용, CLI 및 MCP 서버. MCP 경로의 에이전트는 4배 더 적은 출력 토큰으로 최대 7배 더 빠르게 완료되었습니다. 전체 결과는 대화형 차트와 함께 홈페이지에 게시됩니다.", + "changelogRelease20260612Summary": "이 릴리스에서는 AI 에이전트가 Better Fullstack을 사용하여 스캐폴드하고 결과를 벤치마크 페이지에 게시하는 방법을 벤치마킹하고, 새 스택 그래프에 .NET을 일류 생태계로 추가하고, 훨씬 더 간결한 설치를 제공합니다. 또한 벤치마크 자체에서 발견한 4가지 스캐폴드 버그도 수정합니다.", + "changelogRelease20260612HighlightBenchmark": "동일한 프로젝트 사양을 세 가지 방식으로 스캐폴딩하는 벤치마킹된 프론티어 모델: 프롬프트 전용, CLI 및 MCP 서버. MCP 경로의 에이전트는 4배 더 적은 출력 토큰으로 최대 7배 더 빠르게 완료되었습니다. 전체 결과는 당시 대화형 차트와 함께 벤치마크 페이지에 게시되었습니다.", "changelogRelease20260612HighlightMcp": "Claude Code, Codex, Gemini CLI, Cursor, VS Code, Claude Desktop, Windsurf 및 Zed에 대한 한 번의 붙여넣기 설정으로 MCP 페이지를 재설계했습니다.", "changelogRelease20260612HighlightDotnet": "스택 그래프에 .NET을 최고 수준의 에코시스템으로 추가하고 엔터프라이즈 계층, backend-utils 및 Render/Netlify 배포 옵션을 추가했습니다.", "changelogRelease20260612HighlightInstall": "설치 크기를 42%(122MB → 71MB) 줄이고 웹 항목 청크를 32% 줄였습니다.", @@ -211,16 +211,15 @@ "mcpWorkflowTitleB": "빌드됩니다.", "mcpWorkflowDescription": "에이전트는 웹 빌더와 동일한 스키마 및 호환성 규칙을 사용하므로 생성된 명령은 사용자가 시각적으로 구성할 수 있는 명령과 일치합니다.", "mcpTerminalHeader": "에이전트 세션", - "mcpFinalEyebrow": "벤치마크 지원", - "mcpFinalTitle": "2.6배 더 빠르다", - "mcpFinalTitleEmphasis": "프롬프트 전용.", - "mcpFinalDescription": "ScaffBench에서는 MCP 안내에 따라 프로젝트를 생성하는 것이 에이전트에게 프로젝트를 처음부터 직접 작성하도록 요청하는 것보다 더 빠르고 안정적입니다.", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "에이전트의 성적을 보여 주는 기준은", + "mcpFinalTitleEmphasis": "실제 이슈.", + "mcpFinalDescription": "Fixproof는 비공개 및 공개 코드베이스에서 가져온 봉인된 실제 이슈로 코딩 에이전트를 채점하고, 비공개 테스트로 검증합니다.", "mcpViewBenchmark": "벤치마크 보기", "mcpReadDocs": "MCP 문서 읽기", "mcpStatStructuredTools": "구조화된 도구", "mcpStatReadableResources": "읽을 수 있는 리소스", "mcpStatConfigurableOptions": "구성 가능한 옵션", - "mcpStatFasterPromptOnly": "프롬프트 전용보다 빠릅니다.", "mcpCopyAgentConfiguration": "{agent} 구성 복사", "mcpToolGuidanceDescription": "워크플로 규칙, 필드 의미 및 중요 제약 조건", "mcpToolSchemaDescription": "모든 카테고리에 유효한 옵션, 생태계별로 필터링 가능", @@ -245,34 +244,6 @@ "mcpWorkflowCreateNote": "./my-app에 기록됨", "mcpWorkflowDoneName": "스캐폴드 완료", "mcpWorkflowDoneNote": "bun install을 실행하여 완료하세요.", - "llmBenchmarkDescription": "시간, 토큰, 비용 및 결과가 실제로 구축되는지 여부 등 실제 풀스택 스캐폴딩 작업에서 코딩 에이전트를 측정합니다.", - "llmReadBlog": "블로그 읽기", - "llmTryMcp": "MCP를 사용해 보세요.", - "llmBenchmarkMetric": "벤치마크 지표", - "llmScatterAria": "벤치마크 분산형 차트: 각 지점은 하나의 모델이자 생성 경로입니다.", - "llmScatterUnmetered": "이 축에서는 측정되지 않음(차트에서 제외됨): {models}", - "llmFilterModels": "모델 필터링", - "llmModels": "모델", - "llmClaudeSweep": "6월 12일 스윕", - "llmCodexSweep": "6월 10일 스윕", - "llmLightSweep": "6월 12일 라이트 스윕", - "llmPathCliShort": "BF 언급", - "llmPathPromptShort": "프롬프트", - "llmPathMcpDetail": "MCP 도구를 통해 스캐폴딩합니다.", - "llmPathCliDetail": "에이전트는 Better-Fullstack CLI 명령을 구성합니다.", - "llmPathPromptDetail": "Better-Fullstack 없음 - 에이전트가 모든 파일을 직접 작성합니다.", - "llmBuildsPassing": "빌드 통과", - "llmAvgScaffoldTime": "평균 스캐폴드 시간", - "llmOutputTokens": "스캐폴드당 출력 토큰", - "llmFailedBuilds": "실패한 빌드", - "llmSpeed": "속도", - "llmTokens": "토큰", - "llmErrorRate": "오류율", - "llmMostEfficient": "가장 효율적 ↗", - "llmFastReliable": "빠르고 안정적 ​​↗", - "llmAgentTitle": "에이전트에게 빠른 경로를 제공하세요.", - "llmAgentDescription": "하나의 MCP 서버, 벤치마크에서 사용된 모든 사양-스캐폴드 도구. 에이전트를 선택하고 붙여넣으면 완료됩니다.", - "llmAllSupportedClients": "지원되는 모든 클라이언트", "llmCopyAgentSetupCommand": "{agent} 설정 명령 복사", "llmRunInTerminal": "터미널에서 실행", "llmPasteInto": "{target}에 붙여넣기", @@ -473,56 +444,7 @@ "actionsReset": "초기화", "actionsRandom": "무작위", "navAnalytics": "분석", - "runSeoTitle": "ScaffBench를 직접 실행해 보세요.", - "runSeoDescription": "ScaffBench 벤치마크를 로컬에서 재현하려면 하네스를 복제하고, 에이전트 CLI 또는 API 키를 지정한 다음, 생성된 프로젝트가 빌드되는지 여부를 평가하십시오.", - "runHeroEyebrow": "복제하세요", - "runHeroTitleA": "ScaffBench 실행", - "runHeroTitleB": "직접", - "runHeroDescription": "이 도구는 오픈 소스입니다. 클론하고 Claude Code, Codex, opencode, Kilo 또는 Gemini용 Antigravity와 같은 에이전트를 지정하면 각 사양에 대한 스캐폴딩을 생성하고 생성된 프로젝트가 실제로 설치 및 빌드되는지 여부를 평가합니다. 로그인한 CLI 또는 일반 API 키를 사용하여 작업을 실행할 수 있습니다.", - "runHeroQuickstart": "빠른 시작", - "runHeroBrowseReports": "보고서를 살펴보세요", - "runQuickstartEyebrow": "빠른 시작", - "runQuickstartTitle": "세 단계", - "runStepCloneInstall": "복제 및 설치", - "runStepAuth": "에이전트를 인증하세요", - "runStepRun": "벤치마크를 실행하세요", - "runLabelClone": "하네스를 복제하세요", - "runLabelSignin": "에이전트에 로그인하세요", - "runLabelExportKey": "공급자 키를 내보내기", - "runLabelRunAll": "13개 사양을 모두 실행하고 프롬프트 경로를 지정합니다.", - "runLabelTwoPhase": "2단계", - "runAuthCliTab": "로그인된 CLI", - "runAuthApiTab": "API 키", - "runAuthCliDesc": "이미 로그인한 에이전트 CLI(구독/OAuth)를 사용하세요. 한 번만 로그인하면 그 후에는 하네스가 자동으로 제어합니다. 환경에 키가 필요하지 않습니다.", - "runAuthApiDesc": "API 키를 선호하시나요? 공급자 키를 내보내면 동일한 에이전트 CLI에서 해당 키를 기준으로 요금이 청구됩니다. 구독이 필요하지 않습니다. 구독 기반 실행은 게시되지만 API 실행은 테스트되지 않았지만 지원됩니다.", - "runTwoPhaseNote": "검증 과정을 깔끔하게 유지하고 싶으신가요? 그렇다면 두 단계로 나누세요. 먼저 모든 것을 생성한 다음, 생성된 부분만 따로 검증하는 방식입니다.", - "runResultsNotePre": "결과(리더보드, 사양별 패스, 연결된 라이브러리 및 비용)는 출력 디렉터리에 다음과 같은 형식으로 저장됩니다. ", - "runResultsNoteLink": "발표된 보고서", - "runAgentsEyebrow": "에이전트 및 모델", - "runAgentsTitle": "어떤 에이전트든 데려오세요", - "runAgentsDesc": "제공자는 모델 ID에서 추론되므로 하나의 플래그로 모델과 해당 모델을 구동하는 CLI를 모두 선택할 수 있습니다.", - "runColAgent": "에이전트", - "runColModels": "예시 모델", - "runColAuth": "인증", - "runFlagsEyebrow": "플래그", - "runFlagsTitle": "실행을 조정하세요", - "runFlagModel": "실행할 모델(위 표 참조); 공급자는 ID에서 추론됩니다.", - "runFlagEfforts": "모델이 뒷받침하는 추론 노력", - "runFlagPaths": "prompt는 모든 것을 직접 작성하고, mcp는 MCP 도구를 사용하며, cli는 CLI 명령어를 작성합니다.", - "runFlagSpecs": "기본적으로 13개 사양 전체 또는 쉼표로 구분된 사양 ID 하위 집합을 사용할 수 있습니다.", - "runFlagPhase": "실행 과정을 생성 단계와 검증 단계로 나누고, 각 단계는 자체적으로 검증됩니다.", - "runFlagOutDir": "결과가 저장되는 위치; 동일한 디렉터리를 재사용하여 작업을 재개하거나 유효성을 검사합니다.", - "runCtaEyebrow": "비교하다", - "runCtaTitle": "내 기록이 다른 기록들과 어떻게 다른지 확인해 보세요.", - "runCtaDesc": "여러분의 결과는 리더보드와 동일한 형식으로 표시됩니다. 흥미로운 작업을 수행하셨나요? 보고서를 첨부하여 풀 리퀘스트를 열어주세요.", - "runCtaLeaderboard": "순위표를 확인하세요", - "llmRunItYourself": "직접 실행해 보세요", "navBenchmark": "벤치마크", - "benchmarkTeaserTitle": "AI 모델은 당신의 프로젝트를 얼마나 잘 만들까요?", - "benchmarkTeaserTopModels": "상위 모델", - "benchmarkTeaserCta": "전체 벤치마크 보기", - "benchmarkSeoTitle": "ScaffBench - AI 모델은 당신의 프로젝트를 얼마나 잘 만들까요?", - "benchmarkTeaserMcpBody": "차이는 우리의 MCP입니다. 어떤 코딩 에이전트든 Better-Fullstack 도구에 연결하면 작은 무료 모델조차 훨씬 적은 토큰과 단계로 거의 모든 것을 만들어 냅니다.", "navUpdates": "업데이트", "navTemplates": "템플릿", "builderNewFilter": "이번 릴리스의 새 기능", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "풀스택", "homeStarterShapeFrontend": "프런트엔드만", "homeStarterShapeBackend": "백엔드만", - "homeStarterShapeMobile": "모바일 앱" + "homeStarterShapeMobile": "모바일 앱", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof는 비공개 및 공개 코드베이스에서 가져온 봉인된 실제 이슈로 코딩 에이전트를 채점하고, 비공개 테스트로 검증합니다.", + "fixproofSeoTitle": "Fixproof: 봉인된 코딩 에이전트 벤치마크", + "fixproofClaim": "비공개 및 공개 코드베이스의 실제 이슈를 봉인했습니다. 판정은 비공개 테스트가 합니다.", + "fixproofProvenanceSummary": "모든 수치는 전용 Linux 벤치 머신에서 지정된 에이전트 CLI를 무인으로 실행한 기록에서 나옵니다. 채점은 태스크의 기준 커밋을 대상으로 하며, 그 커밋에서는 실패하고 메인테이너의 수정으로는 통과함이 확인된 비공개 테스트가 판정합니다. 손으로 매긴 값은 없습니다.", + "fixproofStatusRunLabel": "드라이런", + "fixproofStatusDateLabel": "날짜", + "fixproofStatusGradedLabel": "채점한 태스크", + "fixproofStatusTrialsLabel": "태스크당 시도", + "fixproofGradedOfTotal": "{total}개 중 {graded}개", + "fixproofBoardHeading": "보드", + "fixproofBoardCaption": "모델마다 한 행입니다. 두 지수 중 어느 쪽으로도 정렬할 수 있습니다. 열의 물음표는 그 열이 무엇을 세는지 설명합니다.", + "fixproofColModel": "모델", + "fixproofColHarness": "하네스", + "fixproofColEffort": "추론 강도", + "fixproofColResolvedIndex": "Resolved 지수", + "fixproofColProgressIndex": "Progress 지수", + "fixproofColSolvedOverGraded": "해결 / 채점", + "fixproofColRegressions": "회귀", + "fixproofColTestEdits": "되돌린 테스트 수정", + "fixproofColClaimedOnly": "주장만 하고 안 함", + "fixproofColMedianMinutes": "중앙값 분", + "fixproofColRunDate": "실행 날짜", + "fixproofColTrials": "시도", + "fixproofDefHarness": "모델을 구동한 에이전트 CLI입니다.", + "fixproofDefResolvedIndex": "모든 비공개 검사를 통과하고 회귀가 없었던 태스크의 비율을 난이도로 가중한 값입니다. 이 페이지의 대표 수치입니다.", + "fixproofDefProgressIndex": "각 태스크의 요구사항 중 베이스 커밋에서 실패하고 에이전트의 패치 이후 통과한 비율을 가중해 구한 뒤, 태스크 사이에서 난이도로 다시 가중한 값입니다. 요구사항의 가중치는 2(핵심), 1 또는 0.5(주변)입니다. 베이스 커밋에서 이미 통과하던 요구사항과 테스트하지 않은 요구사항은 분자와 분모에서 모두 제외합니다.", + "fixproofDefSolvedOverGraded": "지금까지 채점한 태스크 중 완전히 해결한 태스크입니다. 대기 중인 태스크와 제외된 실행은 분모에 넣지 않습니다.", + "fixproofDefRegressions": "패키지의 기존 테스트 스위트가 더 이상 통과하지 않게 된 채점 실행입니다. 대시는 채점된 실행 중 하나 이상에서 회귀 결과를 알 수 없음을 뜻합니다.", + "fixproofDefTestEdits": "에이전트가 테스트 파일에 가한 수정입니다. 하네스가 채점 전에 되돌립니다.", + "fixproofDefClaimedOnly": "에이전트 요약이 디스크에 반영되지 않은 수정을 했다고 주장한 실행입니다.", + "fixproofDefMedianMinutes": "에이전트가 멈추거나 30분 제한에 도달할 때까지 작업한 실제 시간의 중앙값입니다.", + "fixproofDefTrials": "태스크당 실행 횟수입니다. 한 번의 시도는 표본 하나이므로 작은 차이는 잡음으로 보세요.", + "fixproofSortAria": "{column} 기준 정렬", + "fixproofDefinitionAria": "{column}의 의미는 무엇인가요?", + "fixproofChartHeading": "시간과 지수", + "fixproofChartCaption": "모델과 추론 강도 조합마다 점 하나입니다. 가로축은 왼쪽이 느리고 오른쪽이 빠르므로 좋은 실행일수록 오른쪽 위에 놓입니다.", + "fixproofChartRegionAria": "Fixproof 산점도", + "fixproofChartMetricAria": "차트 지표", + "fixproofChartLegendAria": "벤더", + "fixproofChartAxisMinutes": "태스크당 에이전트 소요 시간 중앙값 (분)", + "fixproofChartNote": "빠를수록 + 높을수록 ↗", + "fixproofChartPointAria": "{model}, {metric} {value}, 중앙값 {minutes}분" } diff --git a/apps/web/messages/uk.json b/apps/web/messages/uk.json index 27e86cb83..5617254bf 100644 --- a/apps/web/messages/uk.json +++ b/apps/web/messages/uk.json @@ -133,8 +133,8 @@ "changelogAllReleases": "Усі релізи", "changelogLatest": "Останній", "changelogRelease20260612Title": "Бенчмарк агентів, екосистема .NET і встановлення на 42% легше", - "changelogRelease20260612Summary": "У цьому релізі ми виміряли, як AI-агенти генерують проєкти з Better Fullstack, опублікували результати на головній, додали .NET як повноцінну екосистему в граф стеку й суттєво полегшили встановлення. Також виправили чотири помилки скафолдингу, які сам бенчмарк і виявив.", - "changelogRelease20260612HighlightBenchmark": "Порівняли frontier-моделі, які генерують той самий проєкт трьома шляхами: лише prompt, наш CLI і MCP-сервер. На MCP-шляху агенти завершували до 7× швидше з 4× меншою кількістю output-токенів; повні результати є на головній з інтерактивним графіком.", + "changelogRelease20260612Summary": "У цьому релізі ми виміряли, як AI-агенти генерують проєкти з Better Fullstack, опублікували результати на сторінці бенчмарка, додали .NET як повноцінну екосистему в граф стеку й суттєво полегшили встановлення. Також виправили чотири помилки скафолдингу, які сам бенчмарк і виявив.", + "changelogRelease20260612HighlightBenchmark": "Порівняли frontier-моделі, які генерують той самий проєкт трьома шляхами: лише prompt, наш CLI і MCP-сервер. На MCP-шляху агенти завершували до 7× швидше з 4× меншою кількістю output-токенів; повні результати тоді опублікували на сторінці бенчмарка з інтерактивним графіком.", "changelogRelease20260612HighlightMcp": "Перероблено сторінку MCP з одноразовим налаштуванням для Claude Code, Codex, Gemini CLI, Cursor, VS Code, Claude Desktop, Windsurf і Zed.", "changelogRelease20260612HighlightDotnet": "Додано .NET як першокласну екосистему, а також корпоративний рівень, серверні утиліти та параметри розгортання Render/Netlify на графі стеку.", "changelogRelease20260612HighlightInstall": "Зменшили розмір встановлення на 42% (122 МБ → 71 МБ) і web entry chunk на 32%.", @@ -212,16 +212,15 @@ "mcpWorkflowTitleB": "Він будує.", "mcpWorkflowDescription": "Агенти використовують ту саму схему й правила сумісності, що й вебконструктор, тому згенеровані команди відповідають тому, що користувач може налаштувати візуально.", "mcpTerminalHeader": "сесія агента", - "mcpFinalEyebrow": "підтверджено бенчмарком", - "mcpFinalTitle": "2,6× швидше, ніж", - "mcpFinalTitleEmphasis": "prompt-only.", - "mcpFinalDescription": "У ScaffBench створення проєкту через MCP швидше й надійніше, ніж просити агента писати проєкт вручну з нуля.", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "Подивіться, як агенти показують себе на", + "mcpFinalTitleEmphasis": "реальних помилках.", + "mcpFinalDescription": "Fixproof оцінює агентів для коду на закритих реальних помилках із приватних і публічних кодових баз, перевірених прихованими тестами.", "mcpViewBenchmark": "Переглянути бенчмарк", "mcpReadDocs": "Читати документацію MCP", "mcpStatStructuredTools": "структуровані інструменти", "mcpStatReadableResources": "доступні для читання ресурси", "mcpStatConfigurableOptions": "настроювані параметри", - "mcpStatFasterPromptOnly": "швидше за prompt-only", "mcpCopyAgentConfiguration": "Копіювати конфігурацію {agent}", "mcpToolGuidanceDescription": "Правила workflow, семантика полів і критичні обмеження", "mcpToolSchemaDescription": "Доступні опції для будь-якої категорії з фільтром за екосистемою", @@ -246,34 +245,6 @@ "mcpWorkflowCreateNote": "записано в ./my-app", "mcpWorkflowDoneName": "скафолдинг завершено", "mcpWorkflowDoneNote": "запустіть bun install, щоб завершити", - "llmBenchmarkDescription": "Вимірюємо агентів для коду на реальних задачах фулстек-скафолдингу: час, токени, вартість і чи справді результат збирається.", - "llmReadBlog": "Читати блог", - "llmTryMcp": "Спробувати MCP", - "llmBenchmarkMetric": "Метрика бенчмарку", - "llmScatterAria": "Scatter chart бенчмарку: кожна точка - одна модель і шлях створення", - "llmScatterUnmetered": "Не вимірюється на цій осі (виключено з графіка): {models}", - "llmFilterModels": "Фільтрувати моделі", - "llmModels": "Моделі", - "llmClaudeSweep": "прогін 12 червня", - "llmCodexSweep": "прогін 10 червня", - "llmLightSweep": "легкий прогін 12 червня", - "llmPathCliShort": "BF mention", - "llmPathPromptShort": "Prompt", - "llmPathMcpDetail": "генерує через наші MCP-інструменти", - "llmPathCliDetail": "агент складає команду Better Fullstack CLI", - "llmPathPromptDetail": "без Better Fullstack - агент вручну пише кожен файл", - "llmBuildsPassing": "Успішні збірки", - "llmAvgScaffoldTime": "Середній час скафолдингу", - "llmOutputTokens": "Output-токени на скафолд", - "llmFailedBuilds": "Невдалі збірки", - "llmSpeed": "Швидкість", - "llmTokens": "Токени", - "llmErrorRate": "Частота помилок", - "llmMostEfficient": "найефективніший ↗", - "llmFastReliable": "швидко + надійно ↗", - "llmAgentTitle": "Дайте агенту короткий шлях.", - "llmAgentDescription": "Один MCP-сервер і всі spec-to-scaffold інструменти, які використовував бенчмарк. Оберіть агента, вставте команду - готово.", - "llmAllSupportedClients": "всі підтримувані клієнти", "llmCopyAgentSetupCommand": "Копіювати команду налаштування {agent}", "llmRunInTerminal": "запустіть у терміналі", "llmPasteInto": "вставте у {target}", @@ -473,56 +444,7 @@ "docsSearchSectionsIndexed": "{count} розділів проіндексовано", "actionsReset": "Скинути", "actionsRandom": "Випадковий", - "runSeoTitle": "Запустіть ScaffBench самостійно", - "runSeoDescription": "Відтворіть ScaffBench локально: клонуйте harness, підключіть будь-який agent CLI або API-ключ і перевірте, чи збираються згенеровані проєкти.", - "runHeroEyebrow": "Відтворіть самі", - "runHeroTitleA": "Запустіть ScaffBench", - "runHeroTitleB": "самостійно", - "runHeroDescription": "Harness має відкритий код. Клонуйте його, підключіть будь-якого агента - Claude Code, Codex, opencode, Kilo або Antigravity для Gemini - і він згенерує кожну специфікацію, а потім перевірить, чи встановлюється та збирається проєкт. Працює з авторизованим CLI або звичайним API-ключем.", - "runHeroQuickstart": "Швидкий старт", - "runHeroBrowseReports": "Перегляньте звіти", - "runQuickstartEyebrow": "Швидкий старт", - "runQuickstartTitle": "Три кроки", - "runStepCloneInstall": "Клонуйте та встановіть", - "runStepAuth": "Авторизуйте агента", - "runStepRun": "Запустіть бенчмарк", - "runLabelClone": "клонувати harness", - "runLabelSignin": "увійдіть у свого агента", - "runLabelExportKey": "експортувати ключ провайдера", - "runLabelRunAll": "запустити всі 13 специфікацій, шлях prompt", - "runLabelTwoPhase": "двофазний", - "runAuthCliTab": "Вхід через CLI", - "runAuthApiTab": "Ключ API", - "runAuthCliDesc": "Використовуйте agent CLI, у який ви вже ввійшли (підписка / OAuth). Увійдіть один раз, а далі harness керує запуском - без ключів у середовищі.", - "runAuthApiDesc": "Бажаєте ключ API? Експортуйте ключ постачальника, і той самий агент CLI виставляє рахунки за нього - підписка не потрібна. Ми публікуємо прогони за підпискою; прогони через API не перевірені, але підтримуються.", - "runTwoPhaseNote": "Хочете тримати валідацію чистою? Розділіть запуск на дві фази: спочатку згенеруйте все, потім перевірте окремо:", - "runResultsNotePre": "Результати - leaderboard, проходження за специфікаціями, підключені бібліотеки й вартість - потрапляють у вихідний каталог у тому самому форматі, що й", - "runResultsNoteLink": "опубліковані звіти", - "runAgentsEyebrow": "Агенти та моделі", - "runAgentsTitle": "Підключіть будь-якого агента", - "runAgentsDesc": "Постачальник визначається за ідентифікатором моделі, тому один прапор вибирає як модель, так і CLI, який нею керує.", - "runColAgent": "Агент", - "runColModels": "Приклади моделей", - "runColAuth": "Автентифікація", - "runFlagsEyebrow": "Прапори", - "runFlagsTitle": "Налаштуйте запуск", - "runFlagModel": "модель для запуску (див. таблицю вище); провайдер визначається за ідентифікатором", - "runFlagEfforts": "міркування, якщо модель підтримує це", - "runFlagPaths": "prompt пише все вручну; mcp проходить через MCP-інструменти; cli складає CLI-команду", - "runFlagSpecs": "повний набір із 13 специфікацій за замовчуванням або підмножина специфікацій, розділених комами", - "runFlagPhase": "розділити прогін на фазу генерації та окрему фазу перевірки", - "runFlagOutDir": "куди записуються результати; використовуйте той самий каталог, щоб відновити або перевірити", - "runCtaEyebrow": "Порівняйте", - "runCtaTitle": "Подивіться, як виглядає ваш прогін", - "runCtaDesc": "Ваші числа мають той самий формат, що й leaderboard. Запустили щось цікаве? Відкрийте PR зі звітом.", - "runCtaLeaderboard": "Переглянути leaderboard", - "llmRunItYourself": "Запустіть самостійно", "navBenchmark": "Бенчмарк", - "benchmarkTeaserTitle": "Наскільки добре моделі ШІ створюють ваші проєкти?", - "benchmarkTeaserTopModels": "Найкращі моделі", - "benchmarkTeaserCta": "Переглянути повний бенчмарк", - "benchmarkSeoTitle": "ScaffBench - Наскільки добре моделі ШІ створюють ваші проєкти?", - "benchmarkTeaserMcpBody": "Різниця - у нашому MCP. Спрямуйте будь-якого агента для коду на інструменти Better-Fullstack, і навіть маленька безкоштовна модель збудує майже все - з часткою токенів і кроків.", "navUpdates": "Оновлення", "navTemplates": "Шаблони", "builderNewFilter": "Нове в цьому релізі", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "Full stack", "homeStarterShapeFrontend": "Лише фронтенд", "homeStarterShapeBackend": "Лише бекенд", - "homeStarterShapeMobile": "Мобільний застосунок" + "homeStarterShapeMobile": "Мобільний застосунок", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof оцінює агентів для коду на закритих реальних помилках із приватних і публічних кодових баз, перевірених прихованими тестами.", + "fixproofSeoTitle": "Fixproof: закритий бенчмарк для агентів для коду", + "fixproofClaim": "Реальні помилки з приватних і публічних кодових баз, закриті. Вирішують приховані тести.", + "fixproofProvenanceSummary": "Кожне число походить із записаного автономного запуску вказаного агентського CLI на окремій Linux-машині для бенчмарків, проти базового коміту задачі та з оцінюванням прихованими тестами, які були червоними на цьому коміті й зеленими з виправленням мейнтейнерів. Тут немає нічого оціненого вручну.", + "fixproofStatusRunLabel": "Пробний запуск", + "fixproofStatusDateLabel": "Дата", + "fixproofStatusGradedLabel": "Оцінено задач", + "fixproofStatusTrialsLabel": "Спроб на задачу", + "fixproofGradedOfTotal": "{graded} з {total}", + "fixproofBoardHeading": "Таблиця", + "fixproofBoardCaption": "Один рядок на модель. Сортування за будь-яким з індексів. Знак питання біля колонки пояснює, що вона рахує.", + "fixproofColModel": "Модель", + "fixproofColHarness": "Harness", + "fixproofColEffort": "Рівень зусиль", + "fixproofColResolvedIndex": "Індекс Resolved", + "fixproofColProgressIndex": "Індекс Progress", + "fixproofColSolvedOverGraded": "Вирішено / оцінено", + "fixproofColRegressions": "Регресії", + "fixproofColTestEdits": "Скасовані зміни в тестах", + "fixproofColClaimedOnly": "Заявлено, не зроблено", + "fixproofColMedianMinutes": "Медіана хвилин", + "fixproofColRunDate": "Дата запуску", + "fixproofColTrials": "Спроби", + "fixproofDefHarness": "CLI агента, який керував моделлю.", + "fixproofDefResolvedIndex": "Зважена за складністю частка задач, де пройшли всі приховані перевірки й не виникло регресій. Це головне число.", + "fixproofDefProgressIndex": "Зважена частка вимог кожної задачі, які падали на базовому коміті й проходять після патча агента, далі зважена за складністю по всіх задачах. Кожна вимога має вагу 2 (основна), 1 або 0,5 (периферійна); вимоги, що вже проходили на базовому коміті, та неперевірені вимоги виключаються і з чисельника, і зі знаменника.", + "fixproofDefSolvedOverGraded": "Повністю вирішені задачі з тих, що вже оцінені. Задачі в очікуванні та виключені запуски не входять у знаменник.", + "fixproofDefRegressions": "Оцінені запуски, де наявний набір тестів пакета перестав проходити. Риска означає, що принаймні для одного оціненого запуску немає результату перевірки регресій.", + "fixproofDefTestEdits": "Зміни, які агент вніс у файли тестів. Harness скасовує їх перед оцінюванням.", + "fixproofDefClaimedOnly": "Запуски, де підсумок агента заявляв про зміни, які так і не потрапили на диск.", + "fixproofDefMedianMinutes": "Медіана реального часу в хвилинах, який агент працював, доки не зупинився або не досяг ліміту в 30 хвилин.", + "fixproofDefTrials": "Запусків на задачу. Одна спроба є однією вибіркою, тому невеликі відмінності варто читати як шум.", + "fixproofSortAria": "Сортувати за {column}", + "fixproofDefinitionAria": "Що означає {column}?", + "fixproofChartHeading": "Час і індекс", + "fixproofChartCaption": "Одна точка на модель і рівень зусиль. Хвилини йдуть від повільних ліворуч до швидких праворуч, тож найкращі запуски опиняються вгорі праворуч.", + "fixproofChartRegionAria": "Точкова діаграма Fixproof", + "fixproofChartMetricAria": "Метрика діаграми", + "fixproofChartLegendAria": "Постачальники", + "fixproofChartAxisMinutes": "Медіана хвилин роботи агента на задачу", + "fixproofChartNote": "швидше + вище ↗", + "fixproofChartPointAria": "{model}, {metric} {value}, медіана {minutes} хвилин" } diff --git a/apps/web/messages/zh-Hant.json b/apps/web/messages/zh-Hant.json index 69b6d272c..b8cbc90bd 100644 --- a/apps/web/messages/zh-Hant.json +++ b/apps/web/messages/zh-Hant.json @@ -132,8 +132,8 @@ "changelogAllReleases": "所有版本", "changelogLatest": "最新", "changelogRelease20260612Title": "代理 benchmark、.NET 生態,以及輕 42% 的安裝體積", - "changelogRelease20260612Summary": "這個版本衡量 AI 代理程式如何使用 Better Fullstack 產生 scaffold,並將結果發佈到首頁;同時在新的 stack graph 中加入一等 .NET 生態,並帶來更輕的安裝包。它還修復了 benchmark 本身發現的四個 scaffold 問題。", - "changelogRelease20260612HighlightBenchmark": "用同一組專案 spec 測試前沿模型的三種 scaffold 路徑:純 prompt、我們的 CLI、以及我們的 MCP 伺服器。走 MCP 路徑的代理程式最高快 7×,輸出 tokens 少 4×;完整結果已在首頁透過互動式圖表展示。", + "changelogRelease20260612Summary": "這個版本衡量 AI 代理程式如何使用 Better Fullstack 產生 scaffold,並將結果發佈到 benchmark 頁面;同時在新的 stack graph 中加入一等 .NET 生態,並帶來更輕的安裝包。它還修復了 benchmark 本身發現的四個 scaffold 問題。", + "changelogRelease20260612HighlightBenchmark": "用同一組專案 spec 測試前沿模型的三種 scaffold 路徑:純 prompt、我們的 CLI、以及我們的 MCP 伺服器。走 MCP 路徑的代理程式最高快 7×,輸出 tokens 少 4×;完整結果當時已在 benchmark 頁面透過互動式圖表展示。", "changelogRelease20260612HighlightMcp": "重新設計 MCP 頁面,為 Claude Code、Codex、Gemini CLI、Cursor、VS Code、Claude Desktop、Windsurf 和 Zed 提供一次貼上即可配置的入口。", "changelogRelease20260612HighlightDotnet": "加入一等 .NET 生態,並在 stack graph 中加入 enterprise 層、backend-utils,以及 Render/Netlify 部署選項。", "changelogRelease20260612HighlightInstall": "安裝體積減少 42%(122 MB → 71 MB),web 入口 chunk 減少 32%。", @@ -211,16 +211,15 @@ "mcpWorkflowTitleB": "它來建構。", "mcpWorkflowDescription": "代理程式使用與網頁建構器相同的 schema 和相容性規則,因此產生的命令會與視覺化設定保持一致。", "mcpTerminalHeader": "代理會話", - "mcpFinalEyebrow": "由 benchmark 支撐", - "mcpFinalTitle": "比", - "mcpFinalTitleEmphasis": "純 prompt 快 2.6×。", - "mcpFinalDescription": "在 ScaffBench 中,由 MCP 引導的專案建立比讓代理從零手寫專案更快也更可靠。", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "在真實問題上", + "mcpFinalTitleEmphasis": "查看代理程式的得分。", + "mcpFinalDescription": "Fixproof 用來自私有和公開程式碼庫的封閉真實問題評測程式代理程式,並由隱藏測試驗證。", "mcpViewBenchmark": "看 benchmark", "mcpReadDocs": "閱讀 MCP 文檔", "mcpStatStructuredTools": "結構化工具", "mcpStatReadableResources": "可讀資源", "mcpStatConfigurableOptions": "可配置選項", - "mcpStatFasterPromptOnly": "比純 prompt 更快", "mcpCopyAgentConfiguration": "複製 {agent} 配置", "mcpToolGuidanceDescription": "工作流程規則、欄位語意和關鍵約束", "mcpToolSchemaDescription": "任意類別的有效選項,可依生態篩選", @@ -245,34 +244,6 @@ "mcpWorkflowCreateNote": "已寫入 ./my-app", "mcpWorkflowDoneName": "scaffold 完成", "mcpWorkflowDoneNote": "運行 bun install 完成安裝", - "llmBenchmarkDescription": "用真實全端鷹架任務衡量程式設計代理:時間、tokens、成本,以及結果是否真的能建構。", - "llmReadBlog": "閱讀部落格", - "llmTryMcp": "試試 MCP", - "llmBenchmarkMetric": "Benchmark 指標", - "llmScatterAria": "Benchmark 散佈圖:每個點代表一個模型和一種建立路徑", - "llmScatterUnmetered": "此座標軸未計量(未繪製在圖中):{models}", - "llmFilterModels": "篩選模型", - "llmModels": "模型", - "llmClaudeSweep": "6 月 12 日批測", - "llmCodexSweep": "6 月 10 日批測", - "llmLightSweep": "6 月 12 日輕量批測", - "llmPathCliShort": "BF 提及", - "llmPathPromptShort": "Prompt", - "llmPathMcpDetail": "透過我們的 MCP 工具產生 scaffold", - "llmPathCliDetail": "代理程式組合 Better-Fullstack CLI 指令", - "llmPathPromptDetail": "不使用 Better-Fullstack:代理程式手寫每個文件", - "llmBuildsPassing": "透過建構", - "llmAvgScaffoldTime": "平均 scaffold 時間", - "llmOutputTokens": "每次 scaffold 輸出 tokens", - "llmFailedBuilds": "失敗建構", - "llmSpeed": "速度", - "llmTokens": "Tokens", - "llmErrorRate": "錯誤率", - "llmMostEfficient": "最高效 ↗", - "llmFastReliable": "快 + 可靠 ↗", - "llmAgentTitle": "給你的代理一條快路徑。", - "llmAgentDescription": "一個 MCP 伺服器,包含 benchmark 使用的所有 spec-to-scaffold 工具。選擇代理,貼上,就緒。", - "llmAllSupportedClients": "所有支援的客戶端", "llmCopyAgentSetupCommand": "複製 {agent} 設定指令", "llmRunInTerminal": "在終端運行", "llmPasteInto": "貼到 {target}", @@ -473,56 +444,7 @@ "actionsReset": "重置", "actionsRandom": "隨機", "navAnalytics": "分析", - "runSeoTitle": "自行運行 ScaffBench", - "runSeoDescription": "在本地重現 ScaffBench 基準測試:克隆測試框架,將其指向任何代理 CLI 或 API 金鑰,並評估產生的專案是否能夠建置。", - "runHeroEyebrow": "自行重現", - "runHeroTitleA": "運行 ScaffBench", - "runHeroTitleB": "你自己", - "runHeroDescription": "該框架是開源的。複製它,並將其指向任何代理程式--Claude Code、Codex、opencode、Kilo 或 Antigravity for Gemini--它就會為每個規範生成腳手架,然後評估生成的專案是否能夠實際安裝和建置。它支援使用已登入的 CLI 或純 API 金鑰運行。", - "runHeroQuickstart": "快速入門", - "runHeroBrowseReports": "瀏覽報告", - "runQuickstartEyebrow": "快速入門", - "runQuickstartTitle": "三步", - "runStepCloneInstall": "克隆並安裝", - "runStepAuth": "驗證你的代理程式", - "runStepRun": "運行基準測試", - "runLabelClone": "克隆測試框架", - "runLabelSignin": "登入您的代理", - "runLabelExportKey": "導出提供者金鑰", - "runLabelRunAll": "運行所有 13 個測試案例,提示路徑", - "runLabelTwoPhase": "兩階段", - "runAuthCliTab": "已登入 CLI", - "runAuthApiTab": "API金鑰", - "runAuthCliDesc": "使用你已登入的代理程式 CLI(訂閱/OAuth)。只需登入一次,然後該框架即可驅動它--你的環境中無需任何金鑰。", - "runAuthApiDesc": "更傾向於使用 API 金鑰?匯出提供者金鑰,即可使用相同代理 CLI 進行計費-無需訂閱。我們提供訂閱驅動的運行服務;API 運行服務未經測試,但我們提供支援。", - "runTwoPhaseNote": "想要保持驗證過程的簡潔性?那就把它分成兩個階段──先生成所有內容,然後再單獨驗證:", - "runResultsNotePre": "結果--排行榜、各規範的通過情況、已串接的函式庫以及成本--都會落在輸出目錄中,格式比照 ", - "runResultsNoteLink": "已發表的報告", - "runAgentsEyebrow": "代理與模型", - "runAgentsTitle": "帶上任何代理人", - "runAgentsDesc": "提供者由模型 ID 推斷得出,因此一個旗標就能同時選定模型與驅動它的 CLI。", - "runColAgent": "代理人", - "runColModels": "範例模型", - "runColAuth": "身份驗證", - "runFlagsEyebrow": "旗標", - "runFlagsTitle": "調整運行", - "runFlagModel": "要運行的模型(參見上表);提供者由 ID 推斷得出。", - "runFlagEfforts": "推理強度(在模型支援的情況下)", - "runFlagPaths": "prompt 負責手動輸入所有內容;mcp 負責使用 MCP 工具;cli 負責編寫 CLI 指令。", - "runFlagSpecs": "預設情況下,使用完整的 13 個規範套件;或使用以逗號分隔的規範 ID 子集。", - "runFlagPhase": "將運行過程分為生成階段和驗證階段,驗證階段單獨進行驗證。", - "runFlagOutDir": "結果落在哪裡;重複使用同一目錄以繼續或驗證", - "runCtaEyebrow": "比較", - "runCtaTitle": "看看你的成績如何", - "runCtaDesc": "你的統計數據將以與排行榜相同的格式顯示。運行了什麼有趣的東西嗎?提交一個 pull request 來附上你的報告吧。", - "runCtaLeaderboard": "看排行榜", - "llmRunItYourself": "自己運行", "navBenchmark": "基準測試", - "benchmarkTeaserTitle": "AI 模型建構你的專案有多強?", - "benchmarkTeaserTopModels": "頂尖模型", - "benchmarkTeaserCta": "查看完整基準測試", - "benchmarkSeoTitle": "ScaffBench - AI 模型建構你的專案有多強?", - "benchmarkTeaserMcpBody": "差別在於我們的 MCP。讓任何程式設計代理接入 Better-Fullstack 的工具,即使是小型免費模型也能用極少的 tokens 和步驟建構幾乎一切。", "navUpdates": "更新", "navTemplates": "範本", "builderNewFilter": "本次發布的新內容", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "全端", "homeStarterShapeFrontend": "僅前端", "homeStarterShapeBackend": "僅後端", - "homeStarterShapeMobile": "行動應用" + "homeStarterShapeMobile": "行動應用", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof 用來自私有和公開程式碼庫的封閉真實問題評測程式代理程式,並由隱藏測試驗證。", + "fixproofSeoTitle": "Fixproof:封閉式程式代理基準測試", + "fixproofClaim": "來自私有和公開程式碼庫的真實問題,全部封閉。由隱藏測試判定。", + "fixproofProvenanceSummary": "每個數字都來自在專用 Linux 基準機器上無人值守執行指定 agent CLI 的記錄,針對任務的基礎 commit,由隱藏測試判定:這些測試在該 commit 上確認為紅,在維護者的修正下確認為綠。這裡沒有任何人工評分。", + "fixproofStatusRunLabel": "試執行", + "fixproofStatusDateLabel": "日期", + "fixproofStatusGradedLabel": "已評測任務", + "fixproofStatusTrialsLabel": "每個任務的試驗次數", + "fixproofGradedOfTotal": "{total} 個中的 {graded} 個", + "fixproofBoardHeading": "榜單", + "fixproofBoardCaption": "每個模型一列。可按任一指數排序。欄位上的問號會說明這一欄統計的是什麼。", + "fixproofColModel": "模型", + "fixproofColHarness": "Harness", + "fixproofColEffort": "推理強度", + "fixproofColResolvedIndex": "Resolved 指數", + "fixproofColProgressIndex": "Progress 指數", + "fixproofColSolvedOverGraded": "已解決 / 已評測", + "fixproofColRegressions": "迴歸", + "fixproofColTestEdits": "已還原的測試改動", + "fixproofColClaimedOnly": "只是聲稱,並未完成", + "fixproofColMedianMinutes": "耗時中位數(分鐘)", + "fixproofColRunDate": "執行日期", + "fixproofColTrials": "試驗次數", + "fixproofDefHarness": "驅動模型的代理程式 CLI。", + "fixproofDefResolvedIndex": "所有隱藏檢查都通過且沒有出現迴歸的任務占比,按難度加權。這是最核心的數字。", + "fixproofDefProgressIndex": "每個任務中,在基線提交上失敗、在代理程式的修補之後通過的需求所占的加權比例,再按難度在任務之間加權。每條需求的權重為 2(核心)、1 或 0.5(外圍);在基線提交上已經通過的需求和未經測試的需求均從分子和分母中排除。", + "fixproofDefSolvedOverGraded": "在目前已評測的任務中完全解決的數量。待執行的任務和被排除的執行不計入分母。", + "fixproofDefRegressions": "評測執行中,套件自帶的測試套件不再通過的那些。 短橫線表示至少一次已評測執行的迴歸結果未知。", + "fixproofDefTestEdits": "代理程式對測試檔案所做的改動。harness 會在評測前把它們還原。", + "fixproofDefClaimedOnly": "代理程式在總結裡聲稱做了改動,但這些改動從未寫入磁碟的執行。", + "fixproofDefMedianMinutes": "代理程式在停止或觸及 30 分鐘上限之前實際工作時長的中位數(分鐘)。", + "fixproofDefTrials": "每個任務的執行次數。一次試驗只是一個樣本,因此細小的差距應當視為雜訊。", + "fixproofSortAria": "按 {column} 排序", + "fixproofDefinitionAria": "{column} 是什麼意思?", + "fixproofChartHeading": "耗時與指數", + "fixproofChartCaption": "每個模型與推理強度組合一個點。橫軸左慢右快,因此表現最好的執行位於右上角。", + "fixproofChartRegionAria": "Fixproof 散佈圖", + "fixproofChartMetricAria": "圖表指標", + "fixproofChartLegendAria": "廠商", + "fixproofChartAxisMinutes": "每個任務的 agent 耗時中位數(分鐘)", + "fixproofChartNote": "更快 + 更高 ↗", + "fixproofChartPointAria": "{model},{metric} {value},中位數 {minutes} 分鐘" } diff --git a/apps/web/messages/zh.json b/apps/web/messages/zh.json index 19c2f239f..09ab94c27 100644 --- a/apps/web/messages/zh.json +++ b/apps/web/messages/zh.json @@ -132,8 +132,8 @@ "changelogAllReleases": "所有版本", "changelogLatest": "最新", "changelogRelease20260612Title": "代理 benchmark、.NET 生态,以及轻 42% 的安装体积", - "changelogRelease20260612Summary": "这个版本衡量 AI 代理如何使用 Better Fullstack 生成 scaffold,并把结果发布到首页;同时在新的 stack graph 中加入一等 .NET 生态,并带来更轻的安装包。它还修复了 benchmark 本身发现的四个 scaffold 问题。", - "changelogRelease20260612HighlightBenchmark": "用同一组项目 spec 测试前沿模型的三种 scaffold 路径:纯 prompt、我们的 CLI、以及我们的 MCP 服务器。走 MCP 路径的代理最高快 7×,输出 tokens 少 4×;完整结果已在首页通过交互图展示。", + "changelogRelease20260612Summary": "这个版本衡量 AI 代理如何使用 Better Fullstack 生成 scaffold,并把结果发布到 benchmark 页面;同时在新的 stack graph 中加入一等 .NET 生态,并带来更轻的安装包。它还修复了 benchmark 本身发现的四个 scaffold 问题。", + "changelogRelease20260612HighlightBenchmark": "用同一组项目 spec 测试前沿模型的三种 scaffold 路径:纯 prompt、我们的 CLI、以及我们的 MCP 服务器。走 MCP 路径的代理最高快 7×,输出 tokens 少 4×;完整结果当时已在 benchmark 页面通过交互图展示。", "changelogRelease20260612HighlightMcp": "重新设计 MCP 页面,为 Claude Code、Codex、Gemini CLI、Cursor、VS Code、Claude Desktop、Windsurf 和 Zed 提供一次粘贴即可配置的入口。", "changelogRelease20260612HighlightDotnet": "加入一等 .NET 生态,并在 stack graph 中加入 enterprise 层、backend-utils,以及 Render/Netlify 部署选项。", "changelogRelease20260612HighlightInstall": "安装体积减少 42%(122 MB → 71 MB),web 入口 chunk 减少 32%。", @@ -211,16 +211,15 @@ "mcpWorkflowTitleB": "它来构建。", "mcpWorkflowDescription": "代理使用与网页构建器相同的 schema 和兼容性规则,因此生成的命令会与可视化配置保持一致。", "mcpTerminalHeader": "代理会话", - "mcpFinalEyebrow": "由 benchmark 支撑", - "mcpFinalTitle": "比", - "mcpFinalTitleEmphasis": "纯 prompt 快 2.6×。", - "mcpFinalDescription": "在 ScaffBench 中,由 MCP 引导的项目创建比让代理从零手写项目更快也更可靠。", + "mcpFinalEyebrow": "Fixproof", + "mcpFinalTitle": "在真实问题上", + "mcpFinalTitleEmphasis": "查看代理的得分。", + "mcpFinalDescription": "Fixproof 用来自私有和公开代码库的封闭真实问题评测编程代理,并由隐藏测试验证。", "mcpViewBenchmark": "查看 benchmark", "mcpReadDocs": "阅读 MCP 文档", "mcpStatStructuredTools": "结构化工具", "mcpStatReadableResources": "可读资源", "mcpStatConfigurableOptions": "可配置选项", - "mcpStatFasterPromptOnly": "比纯 prompt 更快", "mcpCopyAgentConfiguration": "复制 {agent} 配置", "mcpToolGuidanceDescription": "工作流规则、字段语义和关键约束", "mcpToolSchemaDescription": "任意类别的有效选项,可按生态筛选", @@ -245,34 +244,6 @@ "mcpWorkflowCreateNote": "已写入 ./my-app", "mcpWorkflowDoneName": "scaffold 完成", "mcpWorkflowDoneNote": "运行 bun install 完成安装", - "llmBenchmarkDescription": "用真实全栈脚手架任务衡量编程代理:时间、tokens、成本,以及结果是否真的能构建。", - "llmReadBlog": "阅读博客", - "llmTryMcp": "试用 MCP", - "llmBenchmarkMetric": "Benchmark 指标", - "llmScatterAria": "Benchmark 散点图:每个点代表一个模型和一种创建路径", - "llmScatterUnmetered": "此坐标轴未计量(未绘制在图中):{models}", - "llmFilterModels": "筛选模型", - "llmModels": "模型", - "llmClaudeSweep": "6 月 12 日批测", - "llmCodexSweep": "6 月 10 日批测", - "llmLightSweep": "6 月 12 日轻量批测", - "llmPathCliShort": "BF 提及", - "llmPathPromptShort": "Prompt", - "llmPathMcpDetail": "通过我们的 MCP 工具生成 scaffold", - "llmPathCliDetail": "代理组合 Better-Fullstack CLI 命令", - "llmPathPromptDetail": "不使用 Better-Fullstack:代理手写每个文件", - "llmBuildsPassing": "通过构建", - "llmAvgScaffoldTime": "平均 scaffold 时间", - "llmOutputTokens": "每次 scaffold 输出 tokens", - "llmFailedBuilds": "失败构建", - "llmSpeed": "速度", - "llmTokens": "Tokens", - "llmErrorRate": "错误率", - "llmMostEfficient": "最高效 ↗", - "llmFastReliable": "快速 + 可靠 ↗", - "llmAgentTitle": "给你的代理一条快路径。", - "llmAgentDescription": "一个 MCP 服务器,包含 benchmark 使用的所有 spec-to-scaffold 工具。选择代理,粘贴,就绪。", - "llmAllSupportedClients": "所有支持的客户端", "llmCopyAgentSetupCommand": "复制 {agent} 设置命令", "llmRunInTerminal": "在终端运行", "llmPasteInto": "粘贴到 {target}", @@ -473,56 +444,7 @@ "actionsReset": "重置", "actionsRandom": "随机", "navAnalytics": "分析", - "runSeoTitle": "自行运行 ScaffBench", - "runSeoDescription": "在本地重现 ScaffBench 基准测试:克隆测试框架,将其指向任何代理 CLI 或 API 密钥,并评估生成的项目是否能够构建。", - "runHeroEyebrow": "亲自重现", - "runHeroTitleA": "运行 ScaffBench", - "runHeroTitleB": "自己试试", - "runHeroDescription": "该框架是开源的。克隆它,并将其指向任何代理--Claude Code、Codex、opencode、Kilo 或 Antigravity for Gemini--它就会为每个规范生成脚手架,然后评估生成的项目是否能够实际安装和构建。它支持使用已登录的 CLI 或纯 API 密钥运行。", - "runHeroQuickstart": "快速入门", - "runHeroBrowseReports": "浏览报告", - "runQuickstartEyebrow": "快速入门", - "runQuickstartTitle": "三步", - "runStepCloneInstall": "克隆并安装", - "runStepAuth": "认证你的代理", - "runStepRun": "运行基准测试", - "runLabelClone": "克隆测试框架", - "runLabelSignin": "登录你的代理", - "runLabelExportKey": "导出提供商密钥", - "runLabelRunAll": "运行所有 13 个规范,prompt 路径", - "runLabelTwoPhase": "两阶段", - "runAuthCliTab": "已登录 CLI", - "runAuthApiTab": "API 密钥", - "runAuthCliDesc": "使用你已登录的代理 CLI(订阅/OAuth)。只需登录一次,然后该框架即可驱动它--你的环境中无需任何密钥。", - "runAuthApiDesc": "更倾向于使用 API 密钥?导出提供商密钥,即可使用同一代理 CLI 进行计费--无需订阅。我们发布基于订阅的运行结果;API 方式的运行未经测试,但同样受支持。", - "runTwoPhaseNote": "想要保持验证过程的简洁性?那就把它分成两个阶段--先生成所有内容,然后再单独进行验证:", - "runResultsNotePre": "结果--包括排行榜、每个规范的通过情况、已连接的库以及成本--都会保存到输出目录,格式参照", - "runResultsNoteLink": "已发布的报告", - "runAgentsEyebrow": "代理与模型", - "runAgentsTitle": "接入任意代理", - "runAgentsDesc": "提供商由模型 ID 推断得出,因此一个 flag 会同时选定模型和驱动它的 CLI。", - "runColAgent": "代理", - "runColModels": "示例模型", - "runColAuth": "身份验证", - "runFlagsEyebrow": "命令行参数", - "runFlagsTitle": "调整运行", - "runFlagModel": "要运行的模型(参见上表);提供者由 ID 推断得出。", - "runFlagEfforts": "推理强度,在模型支持的情况下", - "runFlagPaths": "prompt 负责手动输入所有内容;mcp 负责使用 MCP 工具;cli 负责编写 CLI 命令。", - "runFlagSpecs": "默认情况下,使用完整的 13 个规范套件;或者使用以逗号分隔的规范 ID 子集。", - "runFlagPhase": "将运行过程分为生成阶段和验证阶段,验证阶段单独进行验证。", - "runFlagOutDir": "结果落在哪里;重用同一目录以继续或验证", - "runCtaEyebrow": "比较", - "runCtaTitle": "看看你的成绩如何", - "runCtaDesc": "你的统计数据将以与排行榜相同的格式显示。运行了什么有趣的东西吗?提交一个 pull request 来附上你的报告吧。", - "runCtaLeaderboard": "查看排行榜", - "llmRunItYourself": "自己运行", "navBenchmark": "基准测试", - "benchmarkTeaserTitle": "AI 模型构建你的项目有多强?", - "benchmarkTeaserTopModels": "顶尖模型", - "benchmarkTeaserCta": "查看完整基准测试", - "benchmarkSeoTitle": "ScaffBench - AI 模型构建你的项目有多强?", - "benchmarkTeaserMcpBody": "差别在于我们的 MCP。让任何编程代理接入 Better-Fullstack 的工具,即使是小型免费模型也能用极少的 tokens 和步骤构建几乎一切。", "navUpdates": "更新", "navTemplates": "模板", "builderNewFilter": "本次发布的新增内容", @@ -576,5 +498,48 @@ "homeStarterShapeFullstack": "全栈", "homeStarterShapeFrontend": "仅前端", "homeStarterShapeBackend": "仅后端", - "homeStarterShapeMobile": "移动应用" + "homeStarterShapeMobile": "移动应用", + "benchmarkTitle": "Fixproof", + "benchmarkDescription": "Fixproof 用来自私有和公开代码库的封闭真实问题评测编程代理,并由隐藏测试验证。", + "fixproofSeoTitle": "Fixproof:封闭式编程代理基准测试", + "fixproofClaim": "来自私有和公开代码库的真实问题,全部封闭。由隐藏测试判定。", + "fixproofProvenanceSummary": "每个数字都来自在专用 Linux 基准机器上无人值守运行指定 agent CLI 的记录,针对任务的基础提交,由隐藏测试判定:这些测试在该提交上确认为红,在维护者的修复下确认为绿。这里没有任何人工打分。", + "fixproofStatusRunLabel": "试运行", + "fixproofStatusDateLabel": "日期", + "fixproofStatusGradedLabel": "已评测任务", + "fixproofStatusTrialsLabel": "每个任务的试验次数", + "fixproofGradedOfTotal": "{total} 个中的 {graded} 个", + "fixproofBoardHeading": "榜单", + "fixproofBoardCaption": "每个模型一行。可按任一指数排序。列上的问号会说明这一列统计的是什么。", + "fixproofColModel": "模型", + "fixproofColHarness": "Harness", + "fixproofColEffort": "推理强度", + "fixproofColResolvedIndex": "Resolved 指数", + "fixproofColProgressIndex": "Progress 指数", + "fixproofColSolvedOverGraded": "已解决 / 已评测", + "fixproofColRegressions": "回归", + "fixproofColTestEdits": "已还原的测试改动", + "fixproofColClaimedOnly": "只是声称,并未完成", + "fixproofColMedianMinutes": "耗时中位数(分钟)", + "fixproofColRunDate": "运行日期", + "fixproofColTrials": "试验次数", + "fixproofDefHarness": "驱动模型的代理 CLI。", + "fixproofDefResolvedIndex": "所有隐藏检查都通过且没有出现回归的任务占比,按难度加权。这是最核心的数字。", + "fixproofDefProgressIndex": "每个任务中,在基线提交上失败、在代理的补丁之后通过的需求所占的加权比例,再按难度在任务之间加权。每条需求的权重为 2(核心)、1 或 0.5(外围);在基线提交上已经通过的需求和未经测试的需求均从分子和分母中排除。", + "fixproofDefSolvedOverGraded": "在目前已评测的任务中完全解决的数量。待运行的任务和被排除的运行不计入分母。", + "fixproofDefRegressions": "评测运行中,包自带的测试套件不再通过的那些。 短横线表示至少一次已评测运行的回归结果未知。", + "fixproofDefTestEdits": "代理对测试文件所做的改动。harness 会在评测前把它们还原。", + "fixproofDefClaimedOnly": "代理在总结里声称做了改动,但这些改动从未落到磁盘上的运行。", + "fixproofDefMedianMinutes": "代理在停止或触及 30 分钟上限之前实际工作时长的中位数(分钟)。", + "fixproofDefTrials": "每个任务的运行次数。一次试验只是一个样本,因此细小的差距应当视为噪声。", + "fixproofSortAria": "按 {column} 排序", + "fixproofDefinitionAria": "{column} 是什么意思?", + "fixproofChartHeading": "耗时与指数", + "fixproofChartCaption": "每个模型与推理强度组合一个点。横轴左慢右快,因此表现最好的运行位于右上角。", + "fixproofChartRegionAria": "Fixproof 散点图", + "fixproofChartMetricAria": "图表指标", + "fixproofChartLegendAria": "厂商", + "fixproofChartAxisMinutes": "每个任务的 agent 耗时中位数(分钟)", + "fixproofChartNote": "更快 + 更高 ↗", + "fixproofChartPointAria": "{model},{metric} {value},中位数 {minutes} 分钟" } diff --git a/apps/web/public/blog-images/scaffbench-2-2.svg b/apps/web/public/blog-images/scaffbench-2-2.svg deleted file mode 100644 index 3596cb788..000000000 --- a/apps/web/public/blog-images/scaffbench-2-2.svg +++ /dev/null @@ -1,107 +0,0 @@ - - ScaffBench 2.2 - Brutalist poster showing full-stack projects moving from prompt to proof through thirteen specifications, three cold trials, and five validation gates. - - - - - - - - - SCAFFBENCH - 2.2 - - - - CODING AGENTS / ZERO-TO-APP - PROMPT - → PROJECT - - ENTIRE FULL-STACK PROJECTS. - VALIDATED COLD. - CLEAN MACHINE / NO CACHE - - - - - 13 SPECS × 3 COLD TRIALS - - - - TRIAL 01 - TRIAL 02 - TRIAL 03 - - - - - 010203 - 040506 - 070809 - 101112 - 13 - - - - PASS - PASS - FAIL - PASS - PASS - FAIL - PASS - PASS - PASS - FAIL - PASS - PASS - PASS - - - PASS - FAIL - PASS - PASS - PASS - PASS - FAIL - PASS - PASS - PASS - FAIL - PASS - PASS - - - PASS - PASS - PASS - FAIL - PASS - PASS - PASS - FAIL - PASS - PASS - PASS - FAIL - PASS - - - - - $ scaffbench run --cold --trials=3 - - INSTALL - - BUILD - - TYPECHECK - - LINT - - TESTS - CLEAN MACHINE / 2026 - - diff --git a/apps/web/src/components/benchmark/fixproof-board.tsx b/apps/web/src/components/benchmark/fixproof-board.tsx new file mode 100644 index 000000000..e338446e3 --- /dev/null +++ b/apps/web/src/components/benchmark/fixproof-board.tsx @@ -0,0 +1,244 @@ +import { useCallback, useMemo, useState, type CSSProperties, type ReactNode } from "react"; +import { TbArrowDown as ArrowDown, TbArrowUp as ArrowUp } from "react-icons/tb"; + +import type { FixproofBoard } from "@/components/benchmark/fixproof-data"; +import type { FixproofRow } from "@/components/benchmark/fixproof-theme"; + +import { + FIXPROOF_CARD, + FIXPROOF_THEME_VARS, + buildRows, + formatMinutes, +} from "@/components/benchmark/fixproof-theme"; +import { ProviderLogo } from "@/components/home/provider-marks"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; +import { cn } from "@/lib/platform/utils"; +import { m } from "@/paraglide/messages.js"; + +type SortKey = "resolvedIndex" | "progressIndex"; + +const HEAD_CELL = + "px-3 py-2.5 font-medium uppercase tracking-[0.14em] text-[10px] text-[var(--fx-label)]"; +const BODY_CELL = "px-3 py-3.5 font-mono text-[13px] tabular-nums"; +const TRACK_STYLE: CSSProperties = { backgroundColor: "var(--fx-track)" }; + +/** A focusable "?" that explains one column without leaving the table. */ +export function MetricHelp({ label, children }: { label: string; children: ReactNode }) { + return ( + + + ? + + +

{label}

+

{children}

+
+
+ ); +} + +function SortHeader({ + label, + help, + sortKey, + active, + descending, + onSort, +}: { + label: string; + help: string; + sortKey: SortKey; + active: boolean; + descending: boolean; + onSort: (key: SortKey) => void; +}) { + const handleClick = useCallback(() => onSort(sortKey), [onSort, sortKey]); + const Arrow = active && !descending ? ArrowUp : ArrowDown; + + return ( + + + + {help} + + + ); +} + +function PlainHeader({ + label, + help, + helpLabel, + align = "right", +}: { + label: string; + help?: string; + /** Title of the tooltip when it explains something narrower than the column. */ + helpLabel?: string; + align?: "left" | "right"; +}) { + return ( + + + {label} + {help ? {help} : null} + + + ); +} + +/** + * An index cell: a bar track in the row's vendor hue, then the number. Progress + * runs at a lower fill so the two index columns stay apart at a glance. + */ +function IndexCell({ value, color, faded }: { value: number; color: string; faded?: boolean }) { + const fillStyle = useMemo( + () => ({ width: `${Math.max(value, 1)}%`, backgroundColor: color, opacity: faded ? 0.45 : 1 }), + [value, color, faded], + ); + + return ( + + + + + + {value} + + + ); +} + +function ModelRow({ row }: { row: FixproofRow }) { + return ( + + + + + {row.label} + + {row.harness} + + {row.effort} + + + + {row.resolved} / {row.graded} + + {row.regressions ?? "–"} + {row.testEditsReverted} + {row.claimedNotDone} + {formatMinutes(row.minutes)} + + {row.runDate} + + {row.trials} + + ); +} + +export function FixproofBoardTable({ board }: { board: FixproofBoard }) { + const [sortKey, setSortKey] = useState("resolvedIndex"); + const [descending, setDescending] = useState(true); + + const onSort = useCallback( + (key: SortKey) => { + if (key === sortKey) { + setDescending((current) => !current); + return; + } + setSortKey(key); + setDescending(true); + }, + [sortKey], + ); + + const rows = useMemo(() => { + const sorted = buildRows(board); + sorted.sort((a, b) => (descending ? b[sortKey] - a[sortKey] : a[sortKey] - b[sortKey])); + return sorted; + }, [board, sortKey, descending]); + + return ( +
+
+ + + + + + + + + + + + + + + + + + + {rows.map((row) => ( + + ))} + +
{m.fixproofBoardCaption()}
+
+
+ ); +} diff --git a/apps/web/src/components/benchmark/fixproof-chart.tsx b/apps/web/src/components/benchmark/fixproof-chart.tsx new file mode 100644 index 000000000..42f8c2b19 --- /dev/null +++ b/apps/web/src/components/benchmark/fixproof-chart.tsx @@ -0,0 +1,520 @@ +import { motion, useInView, useReducedMotion } from "motion/react"; +import { useCallback, useMemo, useRef, useState, type CSSProperties } from "react"; + +import type { FixproofBoard } from "@/components/benchmark/fixproof-data"; +import type { FixproofRow } from "@/components/benchmark/fixproof-theme"; + +import { + FIXPROOF_CARD, + FIXPROOF_THEME_VARS, + buildRows, + formatMinutes, + legendVendors, + pointLabel, +} from "@/components/benchmark/fixproof-theme"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { cn } from "@/lib/platform/utils"; +import { m } from "@/paraglide/messages.js"; + +type ChartMetric = "resolvedIndex" | "progressIndex"; + +interface AxisSpec { + max: number; + ticks: readonly number[]; + label: string; +} + +/** Both indexes run 0 to 100. The headroom keeps a perfect score off the frame. */ +const INDEX_AXIS_MAX = 110; +const INDEX_TICKS: readonly number[] = [0, 25, 50, 75, 100]; + +const VB_W = 1120; +const VB_H = 480; +const M_L = 60; +const M_R = 32; +const M_T = 24; +const M_B = 56; +const PLOT_W = VB_W - M_L - M_R; +const PLOT_H = VB_H - M_T - M_B; + +/** Minutes run fast-to-slow right-to-left, so the best runs sit top right. */ +function plotX(value: number, axis: AxisSpec): number { + return M_L + (1 - value / axis.max) * PLOT_W; +} + +function plotY(value: number): number { + return M_T + (1 - value / INDEX_AXIS_MAX) * PLOT_H; +} + +const barEase = [0.2, 0.8, 0.2, 1] as const; +const chartMove = { duration: 0.7, ease: barEase } as const; +const fadeUpInitial = { opacity: 0, y: 12 } as const; +const fadeUpVisible = { opacity: 1, y: 0 } as const; +const viewportOnceNear = { once: true, margin: "-10%" } as const; +const fadeUpTransition = { duration: 0.6 } as const; + +interface LabelPlacement { + dx?: number; + dy?: number; + anchor?: "start" | "middle" | "end"; + hidden?: boolean; +} + +const PLACEMENT_CANDIDATES: readonly LabelPlacement[] = [ + { anchor: "start", dx: 13, dy: 4.5 }, + { anchor: "end", dx: -13, dy: 4.5 }, + { anchor: "middle", dx: 0, dy: 24 }, + { anchor: "middle", dx: 0, dy: -16 }, + { anchor: "end", dx: -13, dy: 20 }, + { anchor: "end", dx: -13, dy: -12 }, + { anchor: "middle", dx: 0, dy: 38 }, + { anchor: "middle", dx: 0, dy: -30 }, + { anchor: "start", dx: 13, dy: 20 }, + { anchor: "start", dx: 13, dy: -12 }, + { anchor: "end", dx: -13, dy: 34 }, + { anchor: "end", dx: -13, dy: 48 }, + { anchor: "middle", dx: 0, dy: 52 }, + { anchor: "middle", dx: 0, dy: -44 }, +]; + +interface LabelBox { + x1: number; + y1: number; + x2: number; + y2: number; +} + +const LABEL_CHAR_W = 7.5; +const LABEL_ASCENT = 11; +const LABEL_DESCENT = 4; +const DOT_PAD = 12; + +function labelBox(x: number, y: number, width: number, p: LabelPlacement): LabelBox { + const anchorX = x + (p.dx ?? 13); + const x1 = + p.anchor === "end" ? anchorX - width : p.anchor === "middle" ? anchorX - width / 2 : anchorX; + const baseline = y + (p.dy ?? 4.5); + return { x1, y1: baseline - LABEL_ASCENT, x2: x1 + width, y2: baseline + LABEL_DESCENT }; +} + +function boxesOverlap(a: LabelBox, b: LabelBox): boolean { + return a.x1 < b.x2 && b.x1 < a.x2 && a.y1 < b.y2 && b.y1 < a.y2; +} + +function niceStep(maxValue: number): number { + if (maxValue <= 0) return 1; + const target = maxValue / 4; + const magnitude = 10 ** Math.floor(Math.log10(target)); + const normalized = target / magnitude; + let niceNormalized = 1; + if (normalized > 5) { + niceNormalized = 10; + } else if (normalized > 2.5) { + niceNormalized = 5; + } else if (normalized > 1.2) { + niceNormalized = 2; + } + return niceNormalized * magnitude; +} + +function buildMinutesAxis(points: readonly FixproofRow[]): AxisSpec { + const dataMax = Math.max( + 0, + ...points.map((point) => point.minutes).filter((value): value is number => value !== null), + ); + const step = niceStep(dataMax); + const max = Math.max(Math.ceil((dataMax * 1.15) / step) * step, step); + const decimals = step < 1 ? Math.max(0, -Math.floor(Math.log10(step))) : 0; + const ticks: number[] = []; + for (let tick = max; tick >= -1e-9; tick -= step) { + ticks.push(Number(tick.toFixed(decimals))); + } + return { max, ticks, label: m.fixproofChartAxisMinutes() }; +} + +function computeLabelPlacements( + points: readonly FixproofRow[], + axis: AxisSpec, + metric: ChartMetric, +): Record { + const mapped = points.map((point) => ({ + point, + x: plotX(point.minutes ?? 0, axis), + y: plotY(point[metric]), + width: (pointLabel(point).length + 1) * LABEL_CHAR_W, + })); + const obstacles: LabelBox[] = mapped.map((p) => ({ + x1: p.x - DOT_PAD, + y1: p.y - DOT_PAD, + x2: p.x + DOT_PAD, + y2: p.y + DOT_PAD, + })); + obstacles.push({ + x1: M_L + PLOT_W - 8 - 18 * 6.4, + y1: M_T + 6, + x2: M_L + PLOT_W - 8, + y2: M_T + 22, + }); + + const placements: Record = {}; + const ordered = [...mapped].sort((a, b) => b.x - a.x || a.y - b.y); + for (const p of ordered) { + let placed: LabelPlacement = { hidden: true }; + for (const candidate of PLACEMENT_CANDIDATES) { + const box = labelBox(p.x, p.y, p.width, candidate); + if (box.x1 < M_L || box.x2 > VB_W - 2 || box.y1 < 12 || box.y2 > M_T + PLOT_H + 16) continue; + if (obstacles.some((o) => boxesOverlap(box, o))) continue; + placed = candidate; + obstacles.push(box); + break; + } + placements[p.point.key] = placed; + } + return placements; +} + +function metricName(metric: ChartMetric): string { + return metric === "resolvedIndex" ? m.fixproofColResolvedIndex() : m.fixproofColProgressIndex(); +} + +function AxisLayer({ axis, note }: { axis: AxisSpec; note: string }) { + return ( + + {axis.ticks.map((tick) => { + const tx = plotX(tick, axis); + return ( + + + + {tick} + + + ); + })} + {INDEX_TICKS.map((tick) => { + const y = plotY(tick); + return ( + + + + {tick} + + + ); + })} + + {note} + + + {axis.label} + + + ); +} + +function HoverGuides({ active, hex, x, y }: { active: boolean; hex: string; x: number; y: number }) { + return ( + + + + + ); +} + +function ChartMarker({ hex, active }: { hex: string; active: boolean }) { + return ( + <> + + {active ? : null} + + + ); +} + +function ModelDot({ + point, + metric, + x, + y, + placement, + index, + inView, + reduceMotion, + active, + onActiveChange, +}: { + point: FixproofRow; + metric: ChartMetric; + x: number; + y: number; + placement: LabelPlacement | undefined; + index: number; + inView: boolean; + reduceMotion: boolean; + active: boolean; + onActiveChange: (key: string | null) => void; +}) { + const label = pointLabel(point); + const minutes = formatMinutes(point.minutes); + const nearRightEdge = x > M_L + PLOT_W - 150; + const animate = useMemo(() => ({ x, y, opacity: inView ? 1 : 0 }), [x, y, inView]); + const transition = useMemo( + () => + reduceMotion + ? { duration: 0 } + : { + x: chartMove, + y: chartMove, + opacity: { duration: 0.35, delay: 0.1 + index * 0.08 }, + }, + [index, reduceMotion], + ); + const activate = useCallback(() => onActiveChange(point.key), [onActiveChange, point.key]); + const deactivate = useCallback(() => onActiveChange(null), [onActiveChange]); + + return ( + + + + {placement && !placement.hidden ? ( + + {label} + + ) : active ? ( + + {label} + + ) : null} + + + {point[metric]} + + + {minutes} + + + + ); +} + +function VendorLegendItem({ vendor }: { vendor: FixproofRow }) { + const swatchStyle = useMemo( + () => ({ backgroundColor: vendor.color }), + [vendor.color], + ); + + return ( +
  • + + {vendor.vendorLabel} +
  • + ); +} + +function VendorLegend({ rows }: { rows: readonly FixproofRow[] }) { + const vendors = useMemo(() => legendVendors(rows), [rows]); + + return ( +
      + {vendors.map((vendor) => ( + + ))} +
    + ); +} + +export function FixproofChart({ board }: { board: FixproofBoard }) { + const [metric, setMetric] = useState("resolvedIndex"); + const handleMetricChange = useCallback((value: unknown) => { + if (value === "resolvedIndex" || value === "progressIndex") setMetric(value); + }, []); + const [hovered, setHovered] = useState(null); + const ref = useRef(null); + const inView = useInView(ref, { once: true, margin: "-10%" }); + const reduceMotion = useReducedMotion(); + + const rows = useMemo(() => buildRows(board), [board]); + const points = useMemo(() => rows.filter((row) => row.minutes !== null), [rows]); + const axis = useMemo(() => buildMinutesAxis(points), [points]); + const placements = useMemo( + () => computeLabelPlacements(points, axis, metric), + [points, axis, metric], + ); + + return ( + + + + {metricName("resolvedIndex")} + {metricName("progressIndex")} + + +
    +
    +
    +

    {metricName(metric)}

    + + + {points.map((point, index) => ( + + ))} + +
    +
    + +
    +
    +
    +
    + ); +} diff --git a/apps/web/src/components/benchmark/fixproof-data.ts b/apps/web/src/components/benchmark/fixproof-data.ts new file mode 100644 index 000000000..3307e5825 --- /dev/null +++ b/apps/web/src/components/benchmark/fixproof-data.ts @@ -0,0 +1,405 @@ +/** + * Public Fixproof board data. Everything here is safe to publish: a task is only + * an id, a category, a difficulty tier and whether it came from a private or a + * public repository. Statements, repositories and file paths stay sealed. + * + * The shape is written for N models and N tasks even though the first dry run + * has one model and ten tasks. + */ + +export type FixproofOutcome = + | "solved" + /** Oracle green, but the task's own regression suite went red, so the fix does not count as resolved. */ + | "solved-with-regressions" + | "model-failure" + | "deadline-exhausted" + | "provider-infra" + | "pending"; + +export type FixproofSource = "private" | "public"; +/** Per-requirement outcome; "na" means the requirement was already green at base or untested, so it is excluded. */ +export type FixproofRequirementResult = "pass" | "fail" | "na"; + +export type FixproofCategoryId = + | "port" + | "contract" + | "library-semantics" + | "concurrency" + | "bug" + | "effect-ts"; + +/** Category display names are dataset labels, not UI chrome, so they live here. */ +export interface FixproofCategory { + id: FixproofCategoryId; + label: string; +} + +export interface FixproofTask { + id: string; + category: FixproofCategoryId; + /** Difficulty tier used to weight both indexes. */ + difficulty: number; + source: FixproofSource; + /** Hidden checks the task is graded on. */ + requirements: number; + /** Weight of each requirement in the progress share: 2 core, 1 normal, 0.5 peripheral. */ + requirementWeights: readonly number[]; +} + +export interface FixproofRun { + task: string; + outcome: FixproofOutcome; + /** Weighted share of the task's requirements that went from failing to passing, 0..1: sum of weights of "pass" over sum of weights of "pass" and "fail". */ + progress: number | null; + /** One entry per requirement, in the task's requirement order. */ + requirementResults: readonly FixproofRequirementResult[] | null; + /** "passed/total" over every check the harness ran. */ + checks: string | null; + agentSeconds: number | null; + regressions: boolean | null; + testEditsReverted: number; + claimedNotDone: boolean; +} + +export interface FixproofModel { + id: string; + label: string; + effort: string; + provider: string; + harness: string; + runDate: string; + trials: number; + /** Tasks with a recorded, countable result. */ + graded: number; + resolved: number; + /** Difficulty-weighted, 0..100. */ + resolvedIndex: number; + /** Difficulty-weighted, 0..100. */ + progressIndex: number; + medianAgentSeconds: number; + /** Null when any graded run has an unknown regression result. */ + regressions: number | null; + testEditsReverted: number; + claimedNotDone: number; + infraExcluded: number; + pending: number; + runs: readonly FixproofRun[]; +} + +export interface FixproofProtocol { + agentTimeoutMinutes: number; + trialsPerTask: number; + historyHidden: boolean; + spoilerDocsStripped: boolean; + agentTestEditsReverted: boolean; + indexWeighting: "difficulty"; +} + +export interface FixproofBoard { + version: string; + /** Which dry run this board reports. */ + dryRun: number; + generatedAt: string; + protocol: FixproofProtocol; + categories: readonly FixproofCategory[]; + tasks: readonly FixproofTask[]; + models: readonly FixproofModel[]; +} + +export const FIXPROOF_BOARD: FixproofBoard = { + version: "0.1-dryrun", + dryRun: 1, + generatedAt: "2026-09-04", + protocol: { + agentTimeoutMinutes: 30, + trialsPerTask: 1, + historyHidden: true, + spoilerDocsStripped: true, + agentTestEditsReverted: true, + indexWeighting: "difficulty", + }, + categories: [ + { id: "port", label: "Port" }, + { id: "contract", label: "Contract" }, + { id: "library-semantics", label: "Library semantics" }, + { id: "concurrency", label: "Concurrency" }, + { id: "bug", label: "Bug" }, + { id: "effect-ts", label: "Effect TS" }, + ], + tasks: [ + { id: "T01", category: "port", difficulty: 9, source: "private", requirements: 3, requirementWeights: [2, 1, 0.5] }, + { id: "T02", category: "contract", difficulty: 9, source: "private", requirements: 2, requirementWeights: [2, 1] }, + { id: "T03", category: "library-semantics", difficulty: 9, source: "private", requirements: 6, requirementWeights: [2, 2, 2, 2, 1, 1] }, + { id: "T04", category: "concurrency", difficulty: 9, source: "private", requirements: 4, requirementWeights: [2, 1, 1, 0.5] }, + { id: "T05", category: "bug", difficulty: 9, source: "private", requirements: 3, requirementWeights: [2, 0.5, 1] }, + { id: "T06", category: "concurrency", difficulty: 9, source: "private", requirements: 4, requirementWeights: [2, 1, 1, 1] }, + { id: "T07", category: "effect-ts", difficulty: 9, source: "private", requirements: 3, requirementWeights: [2, 2, 2] }, + { id: "T08", category: "concurrency", difficulty: 9, source: "private", requirements: 6, requirementWeights: [2, 2, 1, 1, 2, 1] }, + { id: "T09", category: "effect-ts", difficulty: 8, source: "private", requirements: 3, requirementWeights: [2, 1, 0.5] }, + { id: "T10", category: "concurrency", difficulty: 9, source: "public", requirements: 5, requirementWeights: [1, 2, 0.5, 1, 1] }, + ], + models: [ + { + id: "gemini-3.8-flash|low", + label: "Gemini 3.8 Flash", + effort: "Low", + provider: "Google", + harness: "Antigravity CLI", + runDate: "2026-09-04", + trials: 1, + graded: 10, + resolved: 2, + resolvedIndex: 19, + progressIndex: 40, + medianAgentSeconds: 1652, + regressions: null, + testEditsReverted: 6, + claimedNotDone: 2, + infraExcluded: 0, + pending: 0, + runs: [ + { + task: "T01", + requirementResults: ["pass", "fail", "pass"], + outcome: "model-failure", + progress: 0.71, + checks: "151/152", + agentSeconds: 1442, + regressions: false, + testEditsReverted: 2, + claimedNotDone: true, + }, + { + task: "T02", + requirementResults: ["pass", "pass"], + outcome: "solved", + progress: 1, + checks: "1/1", + agentSeconds: 1736, + regressions: false, + testEditsReverted: 1, + claimedNotDone: false, + }, + { + task: "T03", + requirementResults: ["fail", "fail", "fail", "pass", "pass", "pass"], + outcome: "model-failure", + progress: 0.4, + checks: "24/28", + agentSeconds: 1228, + regressions: false, + testEditsReverted: 2, + claimedNotDone: false, + }, + { + task: "T04", + requirementResults: ["fail", "fail", "na", "na"], + outcome: "model-failure", + progress: 0, + checks: "1/2", + agentSeconds: 1661, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T05", + requirementResults: ["fail", "fail", "na"], + outcome: "deadline-exhausted", + progress: 0, + checks: "1/2", + agentSeconds: 1804, + regressions: null, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T06", + requirementResults: ["fail", "na", "fail", "na"], + outcome: "deadline-exhausted", + progress: 0, + checks: "1/3", + agentSeconds: 1806, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T07", + requirementResults: ["fail", "fail", "fail"], + outcome: "model-failure", + progress: 0, + checks: "0/3", + agentSeconds: 1323, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T08", + requirementResults: ["pass", "pass", "na", "pass", "pass", "pass"], + outcome: "deadline-exhausted", + progress: 1, + checks: "4/4", + agentSeconds: 1804, + regressions: true, + testEditsReverted: 1, + claimedNotDone: false, + }, + { + task: "T09", + requirementResults: ["pass", "pass", "na"], + outcome: "solved", + progress: 1, + checks: "2/2", + agentSeconds: 1356, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T10", + requirementResults: ["fail", "fail", "fail", "fail", "fail"], + outcome: "model-failure", + progress: 0, + checks: "0/7", + agentSeconds: 1642, + regressions: false, + testEditsReverted: 0, + claimedNotDone: true, + }, + ], + }, + { + id: "gpt-5.6-luna|high", + label: "GPT-5.6 Luna", + effort: "High", + provider: "OpenAI", + harness: "Codex CLI", + runDate: "2026-09-04", + trials: 1, + graded: 10, + resolved: 4, + resolvedIndex: 39, + progressIndex: 65, + medianAgentSeconds: 570, + regressions: null, + testEditsReverted: 1, + claimedNotDone: 1, + infraExcluded: 0, + pending: 0, + runs: [ + { + task: "T01", + requirementResults: ["fail", "fail", "pass"], + outcome: "model-failure", + progress: 0.14, + checks: "125/152", + agentSeconds: 783, + regressions: false, + testEditsReverted: 1, + claimedNotDone: false, + }, + { + task: "T02", + requirementResults: ["pass", "pass"], + outcome: "solved", + progress: 1, + checks: "1/1", + agentSeconds: 512, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T03", + requirementResults: ["pass", "na", "pass", "pass", "fail", "fail"], + outcome: "model-failure", + progress: 0.75, + checks: "23/28", + agentSeconds: 970, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T04", + requirementResults: ["fail", "fail", "na", "na"], + outcome: "model-failure", + progress: 0, + checks: "1/2", + agentSeconds: 526, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T05", + requirementResults: ["fail", "fail", "fail"], + outcome: "model-failure", + progress: 0, + checks: "0/2", + agentSeconds: 444, + regressions: null, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T06", + requirementResults: ["pass", "na", "fail", "na"], + outcome: "model-failure", + progress: 0.67, + checks: "2/3", + agentSeconds: 786, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T07", + requirementResults: ["pass", "pass", "pass"], + outcome: "solved", + progress: 1, + checks: "3/3", + agentSeconds: 571, + regressions: false, + testEditsReverted: 0, + claimedNotDone: true, + }, + { + task: "T08", + requirementResults: ["pass", "pass", "na", "pass", "pass", "pass"], + outcome: "solved-with-regressions", + progress: 1, + checks: "4/4", + agentSeconds: 568, + regressions: true, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T09", + requirementResults: ["pass", "pass", "na"], + outcome: "solved", + progress: 1, + checks: "2/2", + agentSeconds: 290, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + { + task: "T10", + requirementResults: ["pass", "pass", "pass", "pass", "pass"], + outcome: "solved", + progress: 1, + checks: "7/7", + agentSeconds: 805, + regressions: false, + testEditsReverted: 0, + claimedNotDone: false, + }, + ], + }, + ], +}; diff --git a/apps/web/src/components/benchmark/fixproof-theme.ts b/apps/web/src/components/benchmark/fixproof-theme.ts new file mode 100644 index 000000000..85170a78b --- /dev/null +++ b/apps/web/src/components/benchmark/fixproof-theme.ts @@ -0,0 +1,153 @@ +/** + * Shared visual language for the Fixproof board and its scatter chart: one hue + * per model vendor so a lab reads the same in both, plus the card surfaces and + * the derived rows both views plot. + */ + +import type { FixproofBoard, FixproofModel } from "@/components/benchmark/fixproof-data"; +import type { ProviderLogoId } from "@/components/home/provider-marks"; + +import { cn } from "@/lib/platform/utils"; + +/** Vendors that carry their own hue. Anything else lands on the neutral slot. */ +export type FixproofVendor = + | "anthropic" + | "openai" + | "google" + | "zai" + | "moonshot" + | "deepseek" + | "qwen" + | "xai" + | "meta" + | "mistral" + | "other"; + +// Light values are the darkened twins of the dark ones, so both themes are +// stepped against their own card surface rather than flipped. +const VENDOR_VARS = cn( + "[--v-anthropic:#c2410c] [--v-openai:#15803d] [--v-google:#1a73e8] [--v-zai:#0d9488] [--v-moonshot:#dc2626]", + "[--v-deepseek:#6d28d9] [--v-qwen:#0891b2] [--v-xai:#4f46e5] [--v-meta:#4338ca] [--v-mistral:#b45309] [--v-other:#57564f]", + "dark:[--v-anthropic:#fb923c] dark:[--v-openai:#4ade80] dark:[--v-google:#60a5fa] dark:[--v-zai:#5eead4] dark:[--v-moonshot:#f87171]", + "dark:[--v-deepseek:#a78bfa] dark:[--v-qwen:#38bdf8] dark:[--v-xai:#a5b4fc] dark:[--v-meta:#818cf8] dark:[--v-mistral:#fbbf24] dark:[--v-other:#a8a69c]", +); + +export const FIXPROOF_THEME_VARS = cn( + VENDOR_VARS, + "[--fx-track:#ececec] [--fx-rule:#e7e5dd] [--fx-tick:#9c9a93] [--fx-label:#71706a]", + "[--fx-surface:#faf9f5] [--fx-edge:#d9d8d2] [--fx-ink:#1b1a17]", + "dark:[--fx-track:rgba(237,235,228,0.08)] dark:[--fx-rule:rgba(237,235,228,0.09)] dark:[--fx-tick:#6c6a61]", + "dark:[--fx-label:#8f8d84] dark:[--fx-surface:#161614] dark:[--fx-edge:rgba(237,235,228,0.14)] dark:[--fx-ink:#dad8d0]", +); + +/** The card both views sit in: light paper, dark ink, scoped colour scheme. */ +export const FIXPROOF_CARD = cn( + "rounded-2xl border border-[#e1e0d8] bg-[#faf9f5] text-[#1b1a17] [color-scheme:light]", + "dark:border-[rgba(237,235,228,0.10)] dark:bg-[#161614] dark:text-[#dad8d0] dark:[color-scheme:dark]", +); + +const VENDOR_BY_PROVIDER: Record = { + anthropic: "anthropic", + openai: "openai", + google: "google", + zai: "zai", + "z.ai": "zai", + moonshot: "moonshot", + deepseek: "deepseek", + qwen: "qwen", + alibaba: "qwen", + xai: "xai", + meta: "meta", + mistral: "mistral", +}; + +const VENDOR_LOGO: Partial> = { + anthropic: "anthropic", + openai: "openai", + google: "google", + zai: "zai", +}; + +/** One plotted entity: a model at one effort, with everything both views need. */ +export interface FixproofRow { + key: string; + label: string; + effort: string; + harness: string; + vendor: FixproofVendor; + vendorLabel: string; + color: string; + logo?: ProviderLogoId; + resolvedIndex: number; + progressIndex: number; + resolved: number; + graded: number; + regressions: number | null; + testEditsReverted: number; + claimedNotDone: number; + /** Median wall-clock minutes, null when nothing was timed. */ + minutes: number | null; + runDate: string; + trials: number; +} + +function toRow(model: FixproofModel): FixproofRow { + const vendor = VENDOR_BY_PROVIDER[model.provider.trim().toLowerCase()] ?? "other"; + return { + key: model.id, + label: model.label, + effort: model.effort, + harness: model.harness, + vendor, + vendorLabel: model.provider, + color: `var(--v-${vendor})`, + logo: VENDOR_LOGO[vendor], + resolvedIndex: model.resolvedIndex, + progressIndex: model.progressIndex, + resolved: model.resolved, + graded: model.graded, + regressions: model.regressions, + testEditsReverted: model.testEditsReverted, + claimedNotDone: model.claimedNotDone, + minutes: model.medianAgentSeconds > 0 ? model.medianAgentSeconds / 60 : null, + runDate: model.runDate, + trials: model.trials, + }; +} + +export function buildRows(board: FixproofBoard): FixproofRow[] { + return board.models.map(toRow); +} + +/** Vendors present on the board, in first-seen order, for the chart legend. */ +export function legendVendors(rows: readonly FixproofRow[]): FixproofRow[] { + const seen = new Set(); + const unique: FixproofRow[] = []; + for (const row of rows) { + if (seen.has(row.vendor)) continue; + seen.add(row.vendor); + unique.push(row); + } + return unique; +} + +/** Model name and effort, the label a point carries in the scatter. */ +export function pointLabel(row: FixproofRow): string { + return row.effort ? `${row.label} ${row.effort}` : row.label; +} + +export function formatMinutes(minutes: number | null): string { + return minutes === null ? "–" : minutes.toFixed(1); +} + +/** How many of the board's tasks have a countable result on at least one model. */ +export function gradedTaskCount(board: FixproofBoard): number { + return board.tasks.filter((task) => + board.models.some((model) => + model.runs.some( + (run) => + run.task === task.id && run.outcome !== "pending" && run.outcome !== "provider-infra", + ), + ), + ).length; +} diff --git a/apps/web/src/components/docs/mdx/bench-bar-chart.tsx b/apps/web/src/components/docs/mdx/bench-bar-chart.tsx index 18c54a54b..a77f7a983 100644 --- a/apps/web/src/components/docs/mdx/bench-bar-chart.tsx +++ b/apps/web/src/components/docs/mdx/bench-bar-chart.tsx @@ -1,7 +1,7 @@ import type { CSSProperties } from "react"; /** - * Horizontal bar chart for benchmark posts (ScaffBench et al.). Rendered inside + * Horizontal bar chart for benchmark posts. Rendered inside * MDX prose, so it opts out with `not-prose` and carries its own card chrome + * warm-stone / lime theming to match the homepage leaderboard. SSR-safe (pure * markup, no browser APIs). One highlighted bar per chart reads as "the result". diff --git a/apps/web/src/components/docs/mdx/verification-status.tsx b/apps/web/src/components/docs/mdx/verification-status.tsx index 245241195..953baced0 100644 --- a/apps/web/src/components/docs/mdx/verification-status.tsx +++ b/apps/web/src/components/docs/mdx/verification-status.tsx @@ -129,7 +129,7 @@ export function VerificationStatus() {

    Runtime verification applies only to the recorded boundaries and limitations above. It does - not prove behavior outside those assertions. ScaffBench measures model performance and does + not prove behavior outside those assertions. Fixproof measures model performance and does not raise this product evidence level.

    diff --git a/apps/web/src/components/home/benchmark-teaser.tsx b/apps/web/src/components/home/benchmark-teaser.tsx deleted file mode 100644 index af40fd750..000000000 --- a/apps/web/src/components/home/benchmark-teaser.tsx +++ /dev/null @@ -1,207 +0,0 @@ -import type { CSSProperties } from "react"; - -import { Link } from "@tanstack/react-router"; -import { motion } from "motion/react"; -import { TbArrowRight as ArrowRight } from "react-icons/tb"; - -import { ProviderLogo, type ProviderLogoId } from "@/components/home/provider-marks"; -import { SCAFFBENCH3_CELLS, SCAFFBENCH3_MODELS } from "@/components/home/scaffbench-3-board-data"; -import { isFreeModel, type ScaffbenchVendor } from "@/components/home/scaffbench-types"; -import { cn } from "@/lib/platform/utils"; -import { m } from "@/paraglide/messages.js"; - -// The teaser shows the leading row of the live ScaffBench 3 board: how often the -// model's project builds, against its ScaffBench Index, the graded and -// difficulty-weighted score the leaderboard sorts by. Numbers come straight from -// the committed run data, so they can't drift from the full leaderboard. -const VENDOR_LOGO: Partial> = { - anthropic: "anthropic", - openai: "openai", - google: "google", - zai: "zai", -}; - -function mean(values: readonly number[]): number { - return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : Number.NaN; -} - -type BoardLeader = { - label: string; - logo?: ProviderLogoId; - isFree: boolean; - core: number; - score: number; - costUsd: number | null; - minutes: number | null; -}; - -function computeLeader(): BoardLeader | null { - let best: BoardLeader | null = null; - for (const model of SCAFFBENCH3_MODELS) { - const scored = SCAFFBENCH3_CELLS.filter((cell) => cell.modelKey === model.key && cell.scored); - if (scored.length === 0 || model.eligibility !== "ranked") continue; - const trials = scored.reduce((sum, cell) => sum + cell.scoredTrials, 0); - const costs = scored.map((cell) => cell.costUsd).filter((v): v is number => v !== null); - const durations = scored - .map((cell) => cell.durationMs) - .filter((v): v is number => v !== null && v > 0); - const leader: BoardLeader = { - label: model.label, - logo: VENDOR_LOGO[model.vendor], - isFree: isFreeModel(model), - core: Math.round((100 * scored.reduce((sum, c) => sum + c.passCount, 0)) / trials), - score: model.sortIndex, - costUsd: costs.length > 0 ? mean(costs) : null, - minutes: durations.length > 0 ? mean(durations) / 60000 : null, - }; - if ( - !best || - leader.score > best.score || - (leader.score === best.score && leader.core > best.core) - ) { - best = leader; - } - } - return best; -} - -const LEADER = computeLeader(); - -const cardReveal = { opacity: 0, y: 16 } as const; -const cardShown = { opacity: 1, y: 0 } as const; -const cardViewport = { once: true, margin: "-80px" } as const; -const cardTransition = { duration: 0.5, ease: [0.16, 1, 0.3, 1] } as const; - -export default function BenchmarkTeaser() { - return ( -
    -
    -
    -

    - ScaffBench -

    -

    - {m.benchmarkTeaserTitle()} -

    -

    - {m.llmBenchmarkDescription()} -

    -
    - - {m.benchmarkTeaserCta()} - - - - {m.llmTryMcp()} - -
    -
    - - {LEADER ? : null} -
    -
    - ); -} - -function BoardLeaderCard({ leader }: { leader: BoardLeader }) { - return ( - -
    - - - {leader.label} - - - {leader.isFree ? "free model" : "leading model"} - -
    - -
    - - -
    -

    - Over 13 specs -

    - -
    - - -
    -
    - ); -} - -const BAR_TRACK: CSSProperties = { backgroundColor: "var(--bar-track)" }; -const CARD_VARS = "[--bar-track:#ececec] dark:[--bar-track:#edebe414]"; - -function PassBar({ - label, - pass, - accent, -}: { - label: string; - pass: number; - accent: "muted" | "lime"; -}) { - const fillStyle: CSSProperties = { - width: `${pass}%`, - backgroundColor: accent === "lime" ? "#C6E853" : "var(--bar-muted)", - }; - return ( -
    - - {label} - - - - - {pass}% -
    - ); -} - -function StatTile({ value, unit }: { value: string; unit: string }) { - return ( -
    -

    {value}

    -

    {unit}

    -
    - ); -} diff --git a/apps/web/src/components/home/llm-benchmark-section.tsx b/apps/web/src/components/home/llm-benchmark-section.tsx deleted file mode 100644 index 6b29f7846..000000000 --- a/apps/web/src/components/home/llm-benchmark-section.tsx +++ /dev/null @@ -1,1445 +0,0 @@ -import { Link } from "@tanstack/react-router"; -import { motion, useInView, useReducedMotion } from "motion/react"; -import { - Fragment, - useCallback, - useMemo, - useRef, - useState, - type CSSProperties, - type ReactNode, -} from "react"; -import { - TbArrowUpRight as ArrowUpRight, - TbCheck as Check, - TbChevronDown as ChevronDown, - TbCopy as Copy, -} from "react-icons/tb"; - -import type { - ScaffbenchCell, - ScaffbenchHarness, - ScaffbenchModel, - ScaffbenchVendor, -} from "@/components/home/scaffbench-types"; - -import { OpenAIMark, ProviderLogo, type ProviderLogoId } from "@/components/home/provider-marks"; -import { SCAFFBENCH3_CELLS, SCAFFBENCH3_MODELS } from "@/components/home/scaffbench-3-board-data"; -import { isFreeModel } from "@/components/home/scaffbench-types"; -import { AgentCommandTabs } from "@/components/mcp/agent-command-tabs"; -import { SCAFFBENCH3_SPECS } from "@/components/scaffbench/scaffbench-3-data"; -import { - DropdownMenu, - DropdownMenuCheckboxItem, - DropdownMenuContent, - DropdownMenuGroup, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; -import { useTheme } from "@/lib/content/theme"; -import { cn } from "@/lib/platform/utils"; -import { m } from "@/paraglide/messages.js"; - -const fadeUpInitial = { opacity: 0, y: 12 } as const; - -const fadeUpVisible = { opacity: 1, y: 0 } as const; - -const viewportOnceNear = { once: true, margin: "-10%" } as const; - -const fadeUpTransition = { duration: 0.6 } as const; - -const headingStyle: CSSProperties = { - fontSize: "clamp(2.2rem, 6vw, 4rem)", - lineHeight: 0.98, -}; - -/** A run is unmetered when the provider billed nothing for any scored spec. */ -// One color per model vendor, so a lab reads the same in the scatter and the -// leaderboard. Light values are the darkened twins of the dark ones. -const VENDOR_THEME_VARS = cn( - "[--v-anthropic:#c2410c] [--v-openai:#15803d] [--v-google:#1a73e8] [--v-zai:#0d9488] [--v-moonshot:#dc2626]", - "[--v-deepseek:#6d28d9] [--v-qwen:#0891b2] [--v-xai:#4f46e5] [--v-meta:#4338ca] [--v-mistral:#b45309]", - "dark:[--v-anthropic:#fb923c] dark:[--v-openai:#4ade80] dark:[--v-google:#60a5fa] dark:[--v-zai:#5eead4] dark:[--v-moonshot:#f87171]", - "dark:[--v-deepseek:#a78bfa] dark:[--v-qwen:#38bdf8] dark:[--v-xai:#a5b4fc] dark:[--v-meta:#818cf8] dark:[--v-mistral:#fbbf24]", -); - -const LEADERBOARD_THEME_VARS = cn( - VENDOR_THEME_VARS, - "[--bar-track:#ececec] [--row-rule:#e7e5dd]", - "dark:[--bar-track:#edebe40f] dark:[--row-rule:rgba(237,235,228,0.07)]", -); - -const VENDOR_COLOR: Record = { - anthropic: "var(--v-anthropic)", - openai: "var(--v-openai)", - google: "var(--v-google)", - zai: "var(--v-zai)", - moonshot: "var(--v-moonshot)", - deepseek: "var(--v-deepseek)", - qwen: "var(--v-qwen)", - xai: "var(--v-xai)", - meta: "var(--v-meta)", - mistral: "var(--v-mistral)", -}; - -const VENDOR_LOGO: Partial> = { - anthropic: "anthropic", - openai: "openai", - google: "google", - zai: "zai", -}; - -const BAR_TRACK_STYLE: CSSProperties = { backgroundColor: "var(--bar-track)" }; - -// Model name and effort get a fixed, generous column so neither is ever clipped. -const LEADERBOARD_GRID = - "grid grid-cols-[minmax(15rem,19rem)_minmax(8rem,1fr)_5rem_3.5rem_4.5rem_4rem_3.5rem] items-center gap-x-3"; - -const PASS_AXIS_TICKS: readonly number[] = [0, 20, 40, 60, 80, 100] as const; - -interface ModelLeaderRow { - key: string; - label: string; - effort: string; - core: number; - color: string; - logo?: ProviderLogoId; - harness: string; - eligibility: ScaffbenchModel["eligibility"]; - /** ScaffBench Index, 0-100: difficulty-weighted mean of the per-spec graded scores. */ - score: number; - time: string; - costNum: number; - cost: string; - outTok: string; - loc: string; - rank?: number; -} - -function annotateRanks(rows: ModelLeaderRow[]): ModelLeaderRow[] { - let rank = 0; - for (const row of rows) { - row.rank = row.eligibility === "ranked" ? (rank += 1) : undefined; - } - return rows; -} - -function formatPercent(passing: number, total: number): number { - return total === 0 ? 0 : Math.round((100 * passing) / total); -} - -function formatDuration(ms: number): string { - const seconds = ms / 1000; - return seconds < 120 ? `${Math.round(seconds)}s` : `${(seconds / 60).toFixed(1)}m`; -} - -function mean(values: readonly number[]): number { - return values.reduce((sum, value) => sum + value, 0) / values.length; -} - -function sortLeaderRows(rows: ModelLeaderRow[]): ModelLeaderRow[] { - const rankedFirst = (row: ModelLeaderRow) => (row.eligibility === "ranked" ? 0 : 1); - return [...rows].sort( - (a, b) => rankedFirst(a) - rankedFirst(b) || b.score - a.score || a.costNum - b.costNum, - ); -} - -const HARNESS_LABEL: Record = { - claude: "Claude Code", - codex: "Codex", - opencode: "opencode", - kilo: "Kilo", - agy: "Antigravity", - pi: "Pi", -}; - -const BOARD_MODELS: readonly ScaffbenchModel[] = [...SCAFFBENCH3_MODELS].sort( - (a, b) => b.sortIndex - a.sortIndex, -); - -const BOARD_SPECS: readonly string[] = SCAFFBENCH3_SPECS.map((spec) => spec.id); - -function cellsFor(modelKey: string): readonly ScaffbenchCell[] { - return SCAFFBENCH3_CELLS.filter((cell) => cell.modelKey === modelKey); -} - -function coreTally(scored: readonly ScaffbenchCell[]): { successes: number; trials: number } { - return { - successes: scored.reduce((sum, cell) => sum + cell.passCount, 0), - trials: scored.reduce((sum, cell) => sum + cell.scoredTrials, 0), - }; -} - -const SPEC_DIFFICULTY = new Map(SCAFFBENCH3_SPECS.map((spec) => [spec.id, spec.difficulty])); - -function scaffbenchIndex(scored: readonly ScaffbenchCell[]): number { - let sum = 0; - let weight = 0; - for (const cell of scored) { - if (cell.score === null) continue; - const w = SPEC_DIFFICULTY.get(cell.spec) ?? 1; - sum += w * cell.score; - weight += w; - } - return weight === 0 ? 0 : Math.round(sum / weight); -} - -function computeScaffbenchModelRows(specs: ReadonlySet): ModelLeaderRow[] { - const rows = BOARD_MODELS.map((model) => { - const scored = cellsFor(model.key).filter((cell) => cell.scored && specs.has(cell.spec)); - const costs = scored.map((cell) => cell.costUsd).filter((v): v is number => v !== null); - const tokens = scored.map((cell) => cell.outTokens).filter((v): v is number => v !== null); - const durations = scored - .map((cell) => cell.durationMs) - .filter((v): v is number => v !== null && v > 0); - const locValues = scored - .map((cell) => cell.lines) - .filter((v): v is number => v !== null && v > 0); - return { - key: model.key, - label: model.label, - effort: model.effort, - color: VENDOR_COLOR[model.vendor], - logo: VENDOR_LOGO[model.vendor], - harness: HARNESS_LABEL[model.harness], - eligibility: model.eligibility, - score: scaffbenchIndex(scored), - core: formatPercent(coreTally(scored).successes, coreTally(scored).trials), - time: durations.length > 0 ? formatDuration(mean(durations)) : "–", - costNum: costs.length > 0 ? mean(costs) : Number.POSITIVE_INFINITY, - cost: costs.length === 0 || mean(costs) === 0 ? "–" : `$${mean(costs).toFixed(2)}`, - outTok: tokens.length > 0 ? `${(mean(tokens) / 1000).toFixed(0)}k` : "–", - loc: locValues.length > 0 ? `${(mean(locValues) / 1000).toFixed(1)}k` : "–", - }; - }); - return sortLeaderRows(rows); -} - -interface LLMBenchmarkSectionProps { - layout?: "two-column" | "stacked"; - className?: string; -} - -export default function LLMBenchmarkSection({ - layout = "two-column", - className, -}: LLMBenchmarkSectionProps = {}) { - if (layout === "stacked") { - return ( -
    -
    - - - - -
    -
    - ); - } - - return ( -
    -
    -
    - - -
    -
    -
    -
    - -
    -
    -
    - -
    -
    - ); -} - -interface AxisSpec { - max: number; - ticks: readonly number[]; - unit: string; - label: string; -} - -const PASS_AXIS: AxisSpec = { - max: 110, - ticks: [0, 25, 50, 75, 100], - unit: "", - label: "ScaffBench Index", -}; - -const VB_W = 1120; -const VB_H = 480; -const M_L = 60; -const M_R = 32; -const M_T = 24; -const M_B = 56; -const PLOT_W = VB_W - M_L - M_R; -const PLOT_H = VB_H - M_T - M_B; - -function plotX(value: number, axis: AxisSpec): number { - return M_L + (1 - value / axis.max) * PLOT_W; -} - -function plotY(value: number, axis: AxisSpec): number { - return M_T + (1 - value / axis.max) * PLOT_H; -} - -const barEase = [0.2, 0.8, 0.2, 1] as const; -const chartMove = { duration: 0.7, ease: barEase } as const; - -interface ChartPalette { - grid: string; - axisTick: string; - axisLabel: string; - note: string; - circleStroke: string; -} - -const CHART_PALETTE: ChartPalette = { - grid: "var(--ch-grid)", - axisTick: "var(--ch-tick)", - axisLabel: "var(--ch-label)", - note: "var(--ch-note)", - circleStroke: "var(--ch-stroke)", -}; - -const CHART_THEME_VARS = cn( - VENDOR_THEME_VARS, - "[--ch-grid:#ececec] [--ch-tick:#9c9a93] [--ch-label:#71706a] [--ch-note:#9c9a93] [--ch-stroke:#ffffff]", - "dark:[--ch-grid:#edebe414] dark:[--ch-tick:#6c6a61] dark:[--ch-label:#8f8d84] dark:[--ch-note:#8f8d84] dark:[--ch-stroke:#161614]", -); - -interface LabelPlacement { - dx?: number; - dy?: number; - anchor?: "start" | "middle" | "end"; - hidden?: boolean; -} - -const PLACEMENT_CANDIDATES: readonly LabelPlacement[] = [ - { anchor: "start", dx: 13, dy: 4.5 }, - { anchor: "end", dx: -13, dy: 4.5 }, - { anchor: "middle", dx: 0, dy: 24 }, - { anchor: "middle", dx: 0, dy: -16 }, - { anchor: "end", dx: -13, dy: 20 }, - { anchor: "end", dx: -13, dy: -12 }, - { anchor: "middle", dx: 0, dy: 38 }, - { anchor: "middle", dx: 0, dy: -30 }, - { anchor: "start", dx: 13, dy: 20 }, - { anchor: "start", dx: 13, dy: -12 }, - { anchor: "end", dx: -13, dy: 34 }, - { anchor: "end", dx: -13, dy: 48 }, - { anchor: "middle", dx: 0, dy: 52 }, - { anchor: "middle", dx: 0, dy: -44 }, -]; - -interface LabelBox { - x1: number; - y1: number; - x2: number; - y2: number; -} - -const LABEL_CHAR_W = 7.5; -const LABEL_ASCENT = 11; -const LABEL_DESCENT = 4; -const DOT_PAD = 12; - -function labelBox(x: number, y: number, width: number, p: LabelPlacement): LabelBox { - const anchorX = x + (p.dx ?? 13); - const x1 = - p.anchor === "end" ? anchorX - width : p.anchor === "middle" ? anchorX - width / 2 : anchorX; - const baseline = y + (p.dy ?? 4.5); - return { x1, y1: baseline - LABEL_ASCENT, x2: x1 + width, y2: baseline + LABEL_DESCENT }; -} - -function boxesOverlap(a: LabelBox, b: LabelBox): boolean { - return a.x1 < b.x2 && b.x1 < a.x2 && a.y1 < b.y2 && b.y1 < a.y2; -} - -type V2Metric = "tokens" | "cost" | "time" | "lines"; - -interface V2ChartTabSpec { - id: V2Metric; - label: string; - note: string; - unit: string; - axisLabel: string; -} - -const V2_CHART_TABS: readonly V2ChartTabSpec[] = [ - { - id: "tokens", - label: "Tokens", - note: "efficient + reliable ↗", - unit: "k", - axisLabel: "Avg output tokens per scaffold", - }, - { - id: "cost", - label: "Cost", - note: "cheap + reliable ↗", - unit: "", - axisLabel: "Avg cost per scaffold ($)", - }, - { - id: "time", - label: "Time", - note: "fast + reliable ↗", - unit: "m", - axisLabel: "Avg minutes per scaffold", - }, - { - id: "lines", - label: "Code", - note: "lean + reliable ↗", - unit: "k", - axisLabel: "Avg lines of code per scaffold", - }, -] as const; - -interface PathMetrics { - pass: number; - corePct: number; - tokens: number | null; - cost: number | null; - time: number | null; - lines: number | null; - scoredCount: number; -} - -function aggregatePathMetrics(sourceCells: readonly ScaffbenchCell[]): PathMetrics { - const scored = sourceCells.filter((cell) => cell.scored); - const tokens = scored - .map((cell) => cell.outTokens) - .filter((value): value is number => value !== null); - const costs = scored - .map((cell) => cell.costUsd) - .filter((value): value is number => value !== null); - const durations = scored - .map((cell) => cell.durationMs) - .filter((value): value is number => value !== null && value > 0); - const lines = scored - .map((cell) => cell.lines) - .filter((value): value is number => value !== null && value > 0); - const core = coreTally(scored); - return { - pass: scaffbenchIndex(scored), - corePct: formatPercent(core.successes, core.trials), - tokens: tokens.length > 0 ? mean(tokens) / 1000 : null, - cost: costs.length > 0 ? mean(costs) : null, - time: durations.length > 0 ? mean(durations) / 60_000 : null, - lines: lines.length > 0 ? mean(lines) / 1000 : null, - scoredCount: scored.length, - }; -} - -type MetricBearing = { - tokens: number | null; - cost: number | null; - time: number | null; - lines: number | null; -}; - -function v2MetricValue(point: MetricBearing, metric: V2Metric): number | null { - if (metric === "cost") return point.cost; - if (metric === "time") return point.time; - if (metric === "lines") return point.lines; - return point.tokens; -} - -function formatV2Metric(point: MetricBearing, metric: V2Metric): string { - if (metric === "cost") return point.cost === null ? "–" : `$${point.cost.toFixed(2)}`; - if (metric === "time") return point.time === null ? "–" : `${point.time.toFixed(1)} min`; - if (metric === "lines") return point.lines === null ? "–" : `${point.lines.toFixed(1)}k lines`; - return point.tokens === null ? "–" : `${point.tokens.toFixed(1)}k tokens`; -} - -function formatV2MetricCompact(point: MetricBearing, metric: V2Metric): string { - if (metric === "cost") return point.cost === null ? "–" : `$${point.cost.toFixed(2)}`; - if (metric === "time") return point.time === null ? "–" : `${point.time.toFixed(1)}m`; - if (metric === "lines") return point.lines === null ? "–" : `${point.lines.toFixed(1)}k`; - return point.tokens === null ? "–" : `${point.tokens.toFixed(1)}k`; -} - -function niceStep(maxValue: number): number { - if (maxValue <= 0) return 1; - const target = maxValue / 4; - const magnitude = 10 ** Math.floor(Math.log10(target)); - const normalized = target / magnitude; - let niceNormalized = 1; - if (normalized > 5) { - niceNormalized = 10; - } else if (normalized > 2.5) { - niceNormalized = 5; - } else if (normalized > 1.2) { - niceNormalized = 2; - } - return niceNormalized * magnitude; -} - -function buildV2Axis(metric: V2Metric, points: readonly MetricBearing[]): AxisSpec { - const tab = V2_CHART_TABS.find((entry) => entry.id === metric) ?? V2_CHART_TABS[0]; - const dataMax = Math.max( - 0, - ...points - .map((point) => v2MetricValue(point, metric)) - .filter((value): value is number => value !== null), - ); - const step = niceStep(dataMax); - const max = Math.max(Math.ceil((dataMax * 1.15) / step) * step, step); - const ticks: number[] = []; - const decimals = step < 1 ? Math.max(0, -Math.floor(Math.log10(step))) : 0; - for (let tick = max; tick >= -1e-9; tick -= step) { - ticks.push(Number(tick.toFixed(decimals))); - } - return { max, ticks, unit: tab.unit, label: tab.axisLabel }; -} - -interface V2ModelPoint extends PathMetrics { - key: string; - label: string; - reasoning: string; - harness: string; - color: string; - free: boolean; -} - -function v2PointLabel(model: ScaffbenchModel): string { - const effort = model.effort.charAt(0).toUpperCase() + model.effort.slice(1); - return `${model.label} ${effort}`; -} - -function computeV2ModelPoints(): V2ModelPoint[] { - return BOARD_MODELS.map((model) => { - const metrics = aggregatePathMetrics(cellsFor(model.key)); - const free = isFreeModel(model); - return { - key: model.key, - label: v2PointLabel(model), - reasoning: model.effort, - harness: HARNESS_LABEL[model.harness], - color: VENDOR_COLOR[model.vendor], - free, - ...metrics, - }; - }); -} - -function v2PointEligible(point: V2ModelPoint, metric: V2Metric): boolean { - if (point.scoredCount === 0) return false; - if (metric === "cost") { - if (point.free) return false; - return point.cost !== null; - } - if (metric === "time") return point.time !== null; - if (metric === "lines") return point.lines !== null; - return true; -} - -function computeV2LabelPlacements( - points: readonly V2ModelPoint[], - axis: AxisSpec, - metric: V2Metric, -): Record { - const inRange = points.filter((point) => { - const val = v2MetricValue(point, metric); - return val !== null && val <= axis.max; - }); - const mapped = inRange.map((point) => ({ - point, - x: plotX(v2MetricValue(point, metric) ?? 0, axis), - y: plotY(point.pass, PASS_AXIS), - width: (point.label.length + 1) * LABEL_CHAR_W, - })); - const obstacles: LabelBox[] = mapped.map((p) => ({ - x1: p.x - DOT_PAD, - y1: p.y - DOT_PAD, - x2: p.x + DOT_PAD, - y2: p.y + DOT_PAD, - })); - obstacles.push({ - x1: M_L + PLOT_W - 8 - 18 * 6.4, - y1: M_T + 6, - x2: M_L + PLOT_W - 8, - y2: M_T + 22, - }); - - const placements: Record = {}; - const ordered = [...mapped].sort((a, b) => b.x - a.x || a.y - b.y); - for (const p of ordered) { - let placed: LabelPlacement = { hidden: true }; - for (const candidate of PLACEMENT_CANDIDATES) { - const box = labelBox(p.x, p.y, p.width, candidate); - if (box.x1 < M_L || box.x2 > VB_W - 2 || box.y1 < 12 || box.y2 > M_T + PLOT_H + 16) continue; - if (obstacles.some((o) => boxesOverlap(box, o))) continue; - placed = candidate; - obstacles.push(box); - break; - } - placements[p.point.key] = placed; - } - return placements; -} - -function BenchmarkChartCard({ className }: { className?: string } = {}) { - const [metric, setMetric] = useState("tokens"); - const [hoveredModel, setHoveredModel] = useState(null); - const modelPoints = useMemo(() => computeV2ModelPoints(), []); - const eligiblePoints = useMemo( - () => modelPoints.filter((point) => v2PointEligible(point, metric)), - [modelPoints, metric], - ); - const [selectedKeys, setSelectedKeys] = useState(() => - modelPoints.map((point) => point.key), - ); - const toggleModel = useCallback((key: string) => { - setSelectedKeys((prev) => - prev.includes(key) - ? prev.length > 1 - ? prev.filter((k) => k !== key) - : prev - : [...prev, key], - ); - }, []); - const visiblePoints = useMemo( - () => eligiblePoints.filter((point) => selectedKeys.includes(point.key)), - [eligiblePoints, selectedKeys], - ); - const plottedPoints = useMemo( - () => visiblePoints.filter((point) => v2MetricValue(point, metric) !== null), - [visiblePoints, metric], - ); - const unmeteredLabels = useMemo( - () => - visiblePoints - .filter((point) => v2MetricValue(point, metric) === null) - .map((point) => point.label), - [visiblePoints, metric], - ); - const axis = useMemo(() => buildV2Axis(metric, plottedPoints), [metric, plottedPoints]); - const labelPlacements = useMemo( - () => computeV2LabelPlacements(plottedPoints, axis, metric), - [plottedPoints, axis, metric], - ); - const axisNote = (V2_CHART_TABS.find((t) => t.id === metric) ?? V2_CHART_TABS[0]).note; - const ref = useRef(null); - const inView = useInView(ref, { once: true, margin: "-10%" }); - const reduceMotion = useReducedMotion(); - const palette = CHART_PALETTE; - - return ( - -
    -
    - {V2_CHART_TABS.map((t) => ( - - ))} -
    - -
    - -
    -
    -
    -

    ScaffBench Index

    - - - {plottedPoints.map((point, index) => { - const val = v2MetricValue(point, metric); - return ( - - ); - })} - -
    -
    - {unmeteredLabels.length > 0 ? ( -

    - {m.llmScatterUnmetered({ models: unmeteredLabels.join(", ") })} -

    - ) : null} -
    -
    - ); -} - -function AxisLayer({ x, note, palette }: { x: AxisSpec; note: string; palette: ChartPalette }) { - return ( - - {x.ticks.map((tick) => { - const tx = plotX(tick, x); - return ( - - - - {tick} - {x.unit} - - - ); - })} - {PASS_AXIS.ticks.map((tick) => { - const y = plotY(tick, PASS_AXIS); - return ( - - - - {tick} - {PASS_AXIS.unit} - - - ); - })} - - {note} - - - {x.label} - - - ); -} - -function HoverGuides({ - active, - hex, - x, - y, -}: { - active: boolean; - hex: string; - x: number; - y: number; -}) { - return ( - - - - - ); -} - -function ChartMarker({ hex, cardBg, active }: { hex: string; cardBg: string; active?: boolean }) { - return ( - <> - - {active ? : null} - - - ); -} - -function MetricTabButton({ - id, - label, - active, - onSelect, -}: { - id: T; - label: string; - active: boolean; - onSelect: (id: T) => void; -}) { - const handleClick = useCallback(() => { - onSelect(id); - }, [onSelect, id]); - - return ( - - ); -} - -function V2ModelFilter({ - points, - selected, - onToggle, -}: { - points: readonly V2ModelPoint[]; - selected: readonly string[]; - onToggle: (key: string) => void; -}) { - const selectedShown = points.filter((point) => selected.includes(point.key)).length; - return ( - - - {m.llmModels()} - - {selectedShown} - - - - - - {points.map((point) => ( - - ))} - - - - ); -} - -function V2ModelMenuItem({ - point, - checked, - onToggle, -}: { - point: V2ModelPoint; - checked: boolean; - onToggle: (key: string) => void; -}) { - const handleChange = useCallback(() => { - onToggle(point.key); - }, [onToggle, point.key]); - const swatchStyle = useMemo(() => ({ background: point.color }), [point.color]); - - return ( - - - - {point.label} [{point.reasoning}] - - - ); -} - -function V2Dot({ - point, - x, - y, - cardBg, - metricLabel, - xAxisValue, - placement, - index, - inView, - reduceMotion, - active, - onActiveChange, -}: { - point: V2ModelPoint; - x: number; - y: number; - cardBg: string; - metricLabel: string; - xAxisValue: string; - placement: LabelPlacement | undefined; - index: number; - inView: boolean; - reduceMotion: boolean; - active: boolean; - onActiveChange: (key: string | null) => void; -}) { - const nearRightEdge = x > M_L + PLOT_W - 150; - const animate = useMemo(() => ({ x, y, opacity: inView ? 1 : 0 }), [x, y, inView]); - const transition = useMemo( - () => - reduceMotion - ? { duration: 0 } - : { - x: chartMove, - y: chartMove, - opacity: { duration: 0.35, delay: 0.1 + index * 0.08 }, - }, - [index, reduceMotion], - ); - const activate = useCallback(() => onActiveChange(point.key), [onActiveChange, point.key]); - const deactivate = useCallback(() => onActiveChange(null), [onActiveChange]); - - return ( - - - - {placement && !placement.hidden ? ( - - {point.label} - - ) : active ? ( - - {point.label} - - ) : null} - - - {point.pass} - - - {xAxisValue} - - - - ); -} - -function BenchmarkMasthead() { - return ( -
    -
    -
    - -

    - ScaffBench 3 -

    -
    -

    - {m.llmBenchmarkDescription()} -

    -
    - -
    - - {m.llmRunItYourself()} - - - - {m.llmTryMcp()} - - -
    -
    - ); -} - -function StackedMasthead() { - return ( -
    -
    - -

    - ScaffBench -

    -
    -

    - {m.llmBenchmarkDescription()} -

    -
    - - {m.llmRunItYourself()} - - - {m.llmTryMcp()} - -
    -
    - ); -} - -const Masthead = StackedMasthead; - -function ScaffBenchMark({ className }: { className?: string }) { - return ( - - - - - ); -} - -function MetricHelp({ label, children }: { label: string; children: ReactNode }) { - return ( - - - ? - - -

    {label}

    -

    {children}

    -
    -
    - ); -} - -const ALL_SPEC_ROWS: readonly ModelLeaderRow[] = computeScaffbenchModelRows(new Set(BOARD_SPECS)); - -function ScaffbenchLeaderboardCard({ className }: { className?: string } = {}) { - const [selectedModelKeys, setSelectedModelKeys] = useState(() => - BOARD_MODELS.map((model) => model.key), - ); - const modelKeysSet = useMemo(() => new Set(selectedModelKeys), [selectedModelKeys]); - const rows = useMemo( - () => annotateRanks(ALL_SPEC_ROWS.filter((row) => modelKeysSet.has(row.key))), - [modelKeysSet], - ); - - const toggleModel = useCallback((key: string) => { - setSelectedModelKeys((prev) => - prev.includes(key) - ? prev.filter((selectedKey) => selectedKey !== key) - : BOARD_MODELS.filter((model) => model.key === key || prev.includes(model.key)).map( - (model) => model.key, - ), - ); - }, []); - - return ( -
    -
    - -
    - - -
    -
    -
    - Model - - - Index - - Every spec earns a graded score: 0.6 when the project installs, builds, - type-checks and compiles on a clean machine, 0.2 for the share of lint and format - gates that pass, and 0.2 for the stack score: the share of the spec's required - libraries actually wired in (dependencies, imports and files, not names mentioned - in passing), minus every trap or restraint marker it broke, such as a forbidden - ORM or build tool. The index is the mean over 13 specs, weighted by spec - difficulty (1 easy, 2 hard, 3 frontier). Tests, cost, time and lines of code are - shown but never scored. The small +N and the faded extension on a bar show how far - the plain build rate sits above the index. - - - Time - Avg cost - Out tok - - LoC - - Mean lines the model actually wrote per scaffold (lockfiles and binaries - excluded). Not part of any score, two green runs can differ 10x in how much code - they took, and that difference is worth seeing. - - -
    - -
    - {rows.map((row) => ( - - ))} -
    - -
    - -
    - {PASS_AXIS_TICKS.map((tick) => ( - {tick} - ))} -
    - - - - - - -
    -
    -
    -
    -
    - ); -} - -function ModelLeaderRow({ row }: { row: ModelLeaderRow }) { - const fillStyle = useMemo( - () => ({ width: `${row.score}%`, backgroundColor: row.color }), - [row.score, row.color], - ); - const coreStyle = useMemo( - () => ({ width: `${row.core}%`, backgroundColor: row.color }), - [row.core, row.color], - ); - const coreBandStyle = useMemo( - () => ({ left: `${row.score}%`, width: `${Math.max(row.core - row.score, 0)}%` }), - [row.core, row.score], - ); - - return ( -
    - - {row.rank !== undefined ? ( - - {row.rank} - - ) : ( - - – - - )} - - - - {row.label} - - -

    Run through {row.harness}

    -
    -
    - {row.effort ? ( - - [{row.effort}] - - ) : null} -
    - -
    -
    -
    -
    - {row.core > row.score ? ( - - - -

    Built {row.core}%

    -

    - Share of specs whose project installs, builds, type-checks and compiles on a clean - machine. The solid bar is the index: those builds credited for green lint and format - gates and for the share of the required stack they wired, weighted by spec - difficulty. -

    -
    -
    - ) : null} -
    - - - {row.score} - {row.core > row.score ? ( - - +{row.core - row.score} - - ) : null} - - {row.time} - {row.cost} - {row.outTok} - {row.loc} -
    - ); -} - -function ModelPicker({ - models, - selectedKeys, - onToggle, -}: { - models: readonly ScaffbenchModel[]; - selectedKeys: readonly string[]; - onToggle: (key: string) => void; -}) { - const firstFreeModelIndex = models.findIndex((model) => isFreeModel(model)); - - return ( - - - Models - - {selectedKeys.length}/{models.length} - - - - - - - Models - - {models.map((model, index) => ( - - {index === firstFreeModelIndex ? : null} - - - ))} - - - - ); -} - -function ModelPickerItem({ - modelKey, - label, - effort, - checked, - onToggle, -}: { - modelKey: string; - label: string; - effort: string; - checked: boolean; - onToggle: (key: string) => void; -}) { - const handleChange = useCallback(() => { - onToggle(modelKey); - }, [onToggle, modelKey]); - - return ( - - {label} - {effort ? ( - - {effort} - - ) : null} - - ); -} - -function AgentInstallPanel({ className }: { className?: string } = {}) { - return ( - -
    -

    - {m.llmAgentTitle()} -

    -

    - {m.llmAgentDescription()} -

    - - {m.llmAllSupportedClients()} - - -
    - - -
    - ); -} diff --git a/apps/web/src/components/home/provider-marks.tsx b/apps/web/src/components/home/provider-marks.tsx index d1d615390..9eeee3669 100644 --- a/apps/web/src/components/home/provider-marks.tsx +++ b/apps/web/src/components/home/provider-marks.tsx @@ -1,7 +1,5 @@ -// Vendor and harness logomarks shared by the ScaffBench leaderboard -// (llm-benchmark-section) and the homepage benchmark teaser. Kept in their own -// module so the teaser can reuse the exact marks without pulling in the full -// chart component. +// Provider logomarks shared by agent-facing components. Kept in their own module +// so the inline SVGs remain statically renderable. /** Brands we render a logo for: model vendors first, then harnesses. */ export type ProviderLogoId = @@ -78,7 +76,7 @@ export function ZaiMark({ className }: { className?: string }) { ); } -// Official harness logomarks, flattened to currentColor for benchmark rows. +// Official harness logomarks, flattened to currentColor for agent-facing labels. export function OpencodeMark({ className }: { className?: string }) { return ( diff --git a/apps/web/src/components/home/scaffbench-3-board-data.ts b/apps/web/src/components/home/scaffbench-3-board-data.ts deleted file mode 100644 index 589e7f2d8..000000000 --- a/apps/web/src/components/home/scaffbench-3-board-data.ts +++ /dev/null @@ -1,799 +0,0 @@ -// AUTO-GENERATED from the ScaffBench 3 run summaries. Do not edit rows by hand. -// Suite 3.0, harness 3.1.0, prompt path, 1 trial per spec. Only models we have -// actually run appear here; the board renders exactly these rows. -// sortIndex follows the graded index (docs/guidelines/scaffbench-benchmark.md, -// Scoring): 0.6 Core + 0.2 lint/format share + 0.2 stack per spec, weighted by -// spec difficulty. GLM 5.3 Flash is exact (per-gate results in its run report). -// GPT-5.6 Luna is the lower bound (73; upper 83): its cells record Full only, so -// every Core-only spec counts lint and format as red. The row stays exploratory, -// unranked, until the run summary is rebuilt with the current harness and the -// exact per-gate results replace the bound. -// Gemini 3.7 Flash (low) is exact: computed by harness 3.1.0 under validator -// cache v9 from testing/llm-benchmarks/v3/lane-3 on Zorro; agy reports no cost -// or tokens, so those columns are empty by construction. Its ts-svelte-edge-orpc -// LoC is null until the archived project is re-measured with .svelte-kit excluded -// from the walk; the run summary recorded the build output as 220,342 lines. -import type { ScaffbenchCell, ScaffbenchModel } from "@web/components/home/scaffbench-types"; - -export const SCAFFBENCH3_MODELS: readonly ScaffbenchModel[] = [ - { - key: "gpt-5.6-luna|high", - model: "gpt-5.6-luna", - effort: "high", - effectiveReasoning: "high", - harness: "codex", - vendor: "openai", - label: "GPT-5.6 Luna", - sortIndex: 73, - eligibility: "exploratory", - free: false, - }, - { - key: "glm-5.3-flash|high", - model: "glm-5.3-flash", - effort: "high", - effectiveReasoning: "high", - harness: "opencode", - vendor: "zai", - label: "GLM 5.3 Flash", - sortIndex: 58, - eligibility: "exploratory", - free: true, - }, - { - key: "gemini-3.7-flash|low", - model: "gemini-3.7-flash", - effort: "low", - effectiveReasoning: "low", - harness: "agy", - vendor: "google", - label: "Gemini 3.7 Flash", - sortIndex: 63, - eligibility: "ranked", - free: false, - }, -]; - -export const SCAFFBENCH3_CELLS: readonly ScaffbenchCell[] = [ - { - modelKey: "gpt-5.6-luna|high", - spec: "ai-search-workbench", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 96, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 79, - lines: 1197, - costUsd: 0.886501, - outTokens: 35219, - steps: null, - durationMs: 742043, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "dotnet-blazor-cqrs", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 965, - costUsd: 0.663307, - outTokens: 30983, - steps: null, - durationMs: 713291, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "elixir-broadway-absinthe", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 793, - costUsd: 0.93621, - outTokens: 39305, - steps: null, - durationMs: 972411, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "frontier-effect-eventsourcing", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 93, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 79, - lines: 648, - costUsd: 0.482846, - outTokens: 28984, - steps: null, - durationMs: 620917, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "frontier-polyglot-proto", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 20, - lines: 908, - costUsd: 0.267081, - outTokens: 15593, - steps: null, - durationMs: 354734, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "go-realtime-api", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 5380, - costUsd: 0.690543, - outTokens: 33856, - steps: null, - durationMs: 786462, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "java-spring-jooq-keycloak", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 96, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 99, - lines: 830, - costUsd: 0.346041, - outTokens: 27600, - steps: null, - durationMs: 598069, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "multi-dotnet-ops", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 80, - lines: 18880, - costUsd: 0.945963, - outTokens: 34600, - steps: null, - durationMs: 838419, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "multi-ts-go-grpc", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 95, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 79, - lines: 2049, - costUsd: 0.940376, - outTokens: 42900, - steps: null, - durationMs: 1058460, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "python-ingestion-api", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 94, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 99, - lines: 1053, - costUsd: 0.538406, - outTokens: 36698, - steps: null, - durationMs: 732746, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "react-native-expo", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 92, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 18, - lines: 972, - costUsd: 1.148512, - outTokens: 42112, - steps: null, - durationMs: 1016461, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "rust-leptos-axum", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 96, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 79, - lines: 610, - costUsd: 0.339955, - outTokens: 18928, - steps: null, - durationMs: 533080, - }, - { - modelKey: "gpt-5.6-luna|high", - spec: "ts-svelte-edge-orpc", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 80, - lines: 44430, - costUsd: 0.732647, - outTokens: 35561, - steps: null, - durationMs: 915753, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "ai-search-workbench", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 96, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 19, - lines: 5649, - costUsd: 0, - outTokens: 46337, - steps: null, - durationMs: 1889584, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "dotnet-blazor-cqrs", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 1055, - costUsd: 0, - outTokens: 20658, - steps: null, - durationMs: 1354954, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "elixir-broadway-absinthe", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 16430, - costUsd: 0, - outTokens: 56744, - steps: null, - durationMs: 2476053, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "frontier-effect-eventsourcing", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 20, - lines: 1050, - costUsd: 0, - outTokens: 32816, - steps: null, - durationMs: 846922, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "frontier-polyglot-proto", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 20, - lines: 2241, - costUsd: 0, - outTokens: 18107, - steps: null, - durationMs: 720423, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "go-realtime-api", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 90, - lines: 10655, - costUsd: 0, - outTokens: 42082, - steps: null, - durationMs: 1884243, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "java-spring-jooq-keycloak", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 93, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 99, - lines: 1660, - costUsd: 0, - outTokens: 36450, - steps: null, - durationMs: 2916988, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "multi-dotnet-ops", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 94, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 89, - lines: 2263, - costUsd: 0, - outTokens: 36711, - steps: null, - durationMs: 2560663, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "multi-ts-go-grpc", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 95, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 79, - lines: 10240, - costUsd: 0, - outTokens: 38371, - steps: null, - durationMs: 1379896, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "python-ingestion-api", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 94, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 19, - lines: 944, - costUsd: 0, - outTokens: 16545, - steps: null, - durationMs: 2226857, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "react-native-expo", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 20, - lines: 1168, - costUsd: 0, - outTokens: 22713, - steps: null, - durationMs: 953958, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "rust-leptos-axum", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 96, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 89, - lines: 1346, - costUsd: 0, - outTokens: 38981, - steps: null, - durationMs: 1822882, - }, - { - modelKey: "glm-5.3-flash|high", - spec: "ts-svelte-edge-orpc", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 80, - lines: 351004, - costUsd: 0, - outTokens: 33018, - steps: null, - durationMs: 1705743, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "ai-search-workbench", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 85, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 17, - lines: 1796, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 90681, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "dotnet-blazor-cqrs", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 1916, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 291181, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "elixir-broadway-absinthe", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 3806, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 558890, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "frontier-effect-eventsourcing", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 80, - lines: 991, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 140036, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "frontier-polyglot-proto", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 87, - lines: 2023, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 233198, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "go-realtime-api", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 90, - lines: 9261, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 136043, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "java-spring-jooq-keycloak", - scored: true, - corePass: true, - fullPass: true, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 1, - score: 100, - lines: 1132, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 207073, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "multi-dotnet-ops", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 94, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 84, - lines: 2162, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 222737, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "multi-ts-go-grpc", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 91, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 18, - lines: 9804, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 233437, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "python-ingestion-api", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 20, - lines: 1008, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 76121, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "react-native-expo", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 20, - lines: 1500, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 258778, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "rust-leptos-axum", - scored: true, - corePass: true, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 1, - qualityPassCount: 0, - score: 80, - lines: 1328, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 452864, - }, - { - modelKey: "gemini-3.7-flash|low", - spec: "ts-svelte-edge-orpc", - scored: true, - corePass: false, - fullPass: false, - wiredPct: 100, - cmdPct: 100, - trials: 1, - scoredTrials: 1, - passCount: 0, - qualityPassCount: 0, - score: 20, - lines: null, - costUsd: null, - outTokens: null, - steps: null, - durationMs: 526396, - }, -]; diff --git a/apps/web/src/components/home/scaffbench-types.ts b/apps/web/src/components/home/scaffbench-types.ts deleted file mode 100644 index 28660bac1..000000000 --- a/apps/web/src/components/home/scaffbench-types.ts +++ /dev/null @@ -1,67 +0,0 @@ -/** Harness a run went through. Drives the harness label, not the row's brand. */ -export type ScaffbenchHarness = "claude" | "codex" | "opencode" | "kilo" | "agy" | "pi"; - -/** Lab that trained the model. Drives the row logomark and bar color. */ -export type ScaffbenchVendor = - | "anthropic" - | "openai" - | "google" - | "zai" - | "moonshot" - | "deepseek" - | "qwen" - | "xai" - | "meta" - | "mistral"; - -export type ScaffbenchModel = { - /** "|" - joins a model to its cells. */ - key: string; - model: string; - effort: string; - effectiveReasoning: string; - harness: ScaffbenchHarness; - vendor: ScaffbenchVendor; - label: string; - /** overall ScaffBench Index across all scored cells - the group sort key. */ - sortIndex: number; - /** "ranked" needs >=3 consistent trials per cell. */ - eligibility: "ranked" | "exploratory"; - /** Free tier, carried explicitly because the display id can drop the source - * alias (GLM 5.3 Flash ran as the free opencode/x-preview-f-free). */ - free: boolean; -}; - -export type ScaffbenchCell = { - modelKey: string; - spec: string; - /** false when the run was infra-inconclusive - excluded from every rate. */ - scored: boolean; - /** installs, builds, type-checks, compiles. */ - corePass: boolean; - /** core plus lint and format green; tests are reported, not scored. */ - fullPass: boolean; - /** share of the spec's required libraries actually wired into the project. */ - wiredPct: number; - /** graded spec score, 0-100: 0.6 core pass + 0.2 lint/format share + 0.2 wired. - * null when the run predates the graded index. */ - score: number | null; - cmdPct: number; - trials: number; - scoredTrials: number; - passCount: number; - qualityPassCount: number; - /** lines the model wrote, lockfiles and binaries excluded. */ - lines: number | null; - costUsd: number | null; - outTokens: number | null; - /** tool steps in the trajectory; null when the harness did not record them. */ - steps: number | null; - durationMs: number | null; -}; - -/** Free tier comes from row metadata, never from a measured $0: subscription - * adapters (opencode-go/*) also report zero cost for paid models. */ -export function isFreeModel(model: Pick): boolean { - return model.free; -} diff --git a/apps/web/src/components/scaffbench/scaffbench-3-data.ts b/apps/web/src/components/scaffbench/scaffbench-3-data.ts deleted file mode 100644 index 1ee8b9001..000000000 --- a/apps/web/src/components/scaffbench/scaffbench-3-data.ts +++ /dev/null @@ -1,165 +0,0 @@ -// AUTO-GENERATED by scripts/benchmarks/build-scaffbench-3-data.ts, do not edit rows by hand. - -export type Scaffbench3Provider = "claude" | "codex" | "opencode" | "kilo" | "agy" | "pi"; - -export type Scaffbench3SpecCell = { - trials: number; - scored: number; - core: number; - quality: number; - score: number | null; -}; - -export type Scaffbench3Spec = { - id: string; - index: number; - family: string; - title: string; - trap: string | null; - /** index weight, pinned per suite: 1 easy, 2 hard, 3 frontier. */ - difficulty: 1 | 2 | 3; -}; - -export type Scaffbench3Row = { - key: string; - model: string; - label: string; - provider: Scaffbench3Provider; - effort: string; - eligibility: "ranked" | "exploratory"; - trials: number; - topUp: "none" | "uniform" | "partial"; - qualityPasses: number; - corePasses: number; - qualityPassPct: number; - corePassPct: number; - scoredSpecs: number; - wiredPct: number; - index: number; - totalCostUsd: number | null; - avgOutTokens: number | null; - medianMinutes: number | null; - results: Record; -}; - -export const SCAFFBENCH3_META = { - suiteVersion: "3.0", - harnessVersion: "3.1.0", - promptVersion: "2026-08-21-scaffbench-3.1", - validationCacheVersion: 9, - resourceProfileId: "low-2w-v1", - trialsPerSpec: 1, - path: "prompt", - qualityGates: true, - generatedAt: null as string | null, - preview: true, -} as const; - -export const SCAFFBENCH3_SPECS: readonly Scaffbench3Spec[] = [ - { - id: "ai-search-workbench", - difficulty: 2, - index: 1, - family: "TypeScript", - title: "AI support search workbench on the Vite+ toolchain", - trap: "pgvector sits one import away; the spec demands Qdrant + OpenSearch split. Turborepo and Biome are forbidden.", - }, - { - id: "rust-leptos-axum", - difficulty: 1, - index: 2, - family: "Rust", - title: "Feature-flag console: Axum API + Leptos WASM frontend", - trap: "Dioxus and Actix are the familiar swaps; both fail the spec.", - }, - { - id: "python-ingestion-api", - difficulty: 1, - index: 3, - family: "Python", - title: "FastAPI document-ingestion pipeline with LangGraph workers", - trap: "Django REST, Django Ninja, and Flask are forbidden.", - }, - { - id: "go-realtime-api", - difficulty: 1, - index: 4, - family: "Go", - title: "Fleet-tracking API: Chi + Ent + gRPC + NATS", - trap: "Gin, Echo, and Fiber all fail; the router must be Chi.", - }, - { - id: "multi-dotnet-ops", - difficulty: 2, - index: 5, - family: "Multi", - title: "Incident-ops portal: Next.js frontend, ASP.NET Minimal API backend", - trap: null, - }, - { - id: "ts-svelte-edge-orpc", - difficulty: 2, - index: 6, - family: "TypeScript", - title: "Edge link-in-bio app: SvelteKit + Hono + D1 on Workers", - trap: "tRPC does not support Svelte, and a Node server adapter fails the Workers deploy.", - }, - { - id: "dotnet-blazor-cqrs", - difficulty: 2, - index: 7, - family: ".NET", - title: "Operations console: Blazor + Dapper + Duende + HotChocolate", - trap: "EF Core, Serilog, and Hangfire are the defaults, and all three are forbidden.", - }, - { - id: "multi-ts-go-grpc", - difficulty: 2, - index: 8, - family: "Multi", - title: "Live auction dashboard: Nuxt (Vue) over a Go Chi + gRPC backend", - trap: "Gin, Echo, and Fiber all fail on the Go side; so do GORM, Ent, Viper, Redis, and zap.", - }, - { - id: "java-spring-jooq-keycloak", - difficulty: 1, - index: 9, - family: "Java", - title: "Event-driven service: Spring Boot + jOOQ + Keycloak + GraphQL", - trap: "Spring Data JPA is forbidden; the data layer must be jOOQ.", - }, - { - id: "elixir-broadway-absinthe", - difficulty: 2, - index: 10, - family: "Elixir", - title: "Realtime ingestion: Phoenix LiveView + Broadway + Oban + Nx", - trap: "Guardian, not phx.gen.auth; Dialyxir, not Credo.", - }, - { - id: "react-native-expo", - difficulty: 2, - index: 11, - family: "React Native", - title: "Offline habit tracker: Expo Router + Uniwind + MMKV", - trap: "NativeWind is the familiar answer and it fails; styling must be Uniwind.", - }, - { - id: "frontier-polyglot-proto", - difficulty: 3, - index: 12, - family: "Frontier", - title: "One proto contract across Rust gRPC, a Go gateway, and a TS client", - trap: "No scaffolder covers this; pure engineering, prompt-only.", - }, - { - id: "frontier-effect-eventsourcing", - difficulty: 3, - index: 13, - family: "Frontier", - title: "Effect bank ledger: event sourcing, CQRS, tRPC over WebSockets", - trap: "Replayable idempotent projections, not CRUD with extra steps.", - }, -]; - -export const SCAFFBENCH3_ROWS: readonly Scaffbench3Row[] = []; diff --git a/apps/web/src/lib/content/changelog.ts b/apps/web/src/lib/content/changelog.ts index 358db3944..90f988c72 100644 --- a/apps/web/src/lib/content/changelog.ts +++ b/apps/web/src/lib/content/changelog.ts @@ -112,12 +112,10 @@ export const changelogReleases: ChangelogRelease[] = [ publishedAt: "2026-07-29T08:49:44Z", displayDate: "July 29, 2026", href: `${RELEASE_BASE_URL}/v2.3.1`, - title: "ScaffBench 2.2, the Run Before You Clone campaign, and expansion fixes", + title: "The Run Before You Clone campaign and expansion fixes", summary: - "This release republishes the agent benchmark on a rebuilt 2.2 harness with honest ranking, launches the Run Before You Clone campaign, and closes the review and CI findings left by the large library expansion.", + "This release launches the Run Before You Clone campaign and closes the review and CI findings left by the large library expansion.", highlights: [ - "Read a rebuilt ScaffBench 2.2 board: a hardened harness and validator, a code-volume metric, tie-band ranks, hover notes explaining surprising rows, and a methodology card above the results.", - "Compare agents across harnesses, not just models, with official harness logos in the table and harness-plus-model pairing in the graph.", "Try a stack before you clone it through the Run Before You Clone campaign and the reworked Edit & Run surface.", "Install Python projects reliably under Poetry, and get pyright-clean output from PyJWT templates.", "Scaffold without npm 10 install failures, and stay on a working Nuxt 4.4.8 while @nuxt/ui catches up to 4.5.", @@ -171,7 +169,6 @@ export const changelogReleases: ChangelogRelease[] = [ "This patch makes the Kotlin language gate tell the truth in both the CLI and the builder, and removes builder surfaces that promised more than they delivered.", highlights: [ "See the same Kotlin availability rules in the CLI and the web builder, backed by one shared predicate, with JPA entities opened correctly for Kotlin projects.", - "Read a ScaffBench graph that filters honestly, with a version dropdown for switching between boards.", "Work in a cleaner builder after the share modal's plugin section and the presets panel's brief suggester were removed.", ], image: gradientArtwork(), @@ -191,7 +188,6 @@ export const changelogReleases: ChangelogRelease[] = [ "Use release channels predictably: latest and beta selections remain on the channel you chose, while generated commands preserve the details needed to reproduce the same stack.", "Install community capability packs with project-contained writes and reliable failure reporting, making them safer to use in scripts and agent workflows.", "Read localized docs, guides, and posts in the selected language from the first page response, without an English body briefly appearing before the page settles.", - "Explore the new ScaffBench MCP path on the homepage, including DeepSeek V4 Flash results across the core benchmark suite.", "Move a builder stack into Claude Code faster with a copy-ready plugin install command in the share dialog.", "Create fresh projects more reliably across Next.js workspaces, Vinext with Strapi, Upstash Redis, Java testing, and multi-ecosystem commands.", "Anonymous usage reporting now distinguishes successful and failed runs, CLI and MCP entry points, and new versus returning installs while keeping error messages and local paths out of telemetry.", @@ -260,7 +256,7 @@ export const changelogReleases: ChangelogRelease[] = [ "This release turns the stack graph into a more visible source of truth: public verified-combination docs, an API badge endpoint, planner-backed CLI add flows, and tighter generated-project CI coverage. It also ships Supabase Auth for TanStack Start fullstack projects and fixes the final release-blocking generator regressions found by broad smoke coverage.", highlights: [ "Added Supabase Auth support for TanStack Start fullstack projects, including server/browser clients, OAuth callback routing, login and dashboard routes, env typing, user-menu wiring, and cookie preservation.", - "Published verified-combination evidence through docs, generated web data, and a Shields-compatible API endpoint so release claims are backed by smoke, ScaffBench, and release-guard artifacts.", + "Published verified-combination evidence through docs, generated web data, and a Shields-compatible API endpoint so release claims are backed by smoke and release-guard artifacts.", "Routed explicit `create-better-fullstack add` stack flags through the stack-update planner/apply path, with dry-run previews, edited-file blockers, richer graph summaries, and regression coverage.", "Hardened generated GitHub Actions output for graph-selected addons across TypeScript and graph-only Rust, Python, Go, Java, Elixir, and .NET projects.", "Fixed TanStack Start OpenAPI/Kysely smoke failures by always declaring React Query devtools where the base template imports it, and updated the release snapshots.", @@ -279,15 +275,14 @@ export const changelogReleases: ChangelogRelease[] = [ publishedAt: "2026-06-29T20:06:24Z", displayDate: "June 29, 2026", href: `${RELEASE_BASE_URL}/v2.1.3`, - title: "ScaffBench 2 agent benchmark, hardened templates, and reliable payments", + title: "Hardened templates and reliable payments", summary: - "This release rebuilds the AI-agent scaffolding benchmark as ScaffBench 2 with honest, reproducible scoring and a live homepage leaderboard. It also makes every generated template pass its own type-check and format gates, fixes all five payment providers, and repairs a wave of stack combinations across the TypeScript, Rust, Go, Python, Java, and Elixir ecosystems.", + "This release makes every generated template pass its own type-check and format gates, fixes all five payment providers, and repairs a wave of stack combinations across the TypeScript, Rust, Go, Python, Java, and Elixir ecosystems.", highlights: [ - "Rebuilt the AI-agent benchmark as ScaffBench 2: a per-spec solvability gate, reproducibility metadata, pass@k / pass^k scoring, and an honest read-only quality gate, plus opencode/Kilo and GPT/Codex agent adapters and free-tier models on an 8-config homepage leaderboard with Core/Full tabs.", "Made generated templates pass their own type-check and format gates - Biome 2.5 preset, Rust cargo fmt --check + clippy -D warnings, Python ruff, gofmt-clean Go, and the Java Testcontainers 2.x rename - so fresh scaffolds stay green.", "Fixed all five payment providers: added env schema for Dodo, Paddle, and Lemon Squeezy, async Paddle webhook verification, Lemon Squeezy SDK type alignment, and stopped pinning a stale Stripe apiVersion.", "Repaired a batch of stack combos: Nuxt oRPC auth context, Kysely auth schema types, OpenAPI tsconfig base path, Qwik Rolldown chunk names, Solid TanStack Router route tree, and Svelte Better Auth builds.", - "The homepage hero release badge now auto-updates from the latest GitHub release, and the benchmark leaderboard ships with real run data.", + "The homepage hero release badge now auto-updates from the latest GitHub release.", "Pinned MikroORM SQLite to the v7 driver and Deno to 2.8.x, and expanded MCP stack-update coverage to keep generated installs and CI reliable.", ], image: { @@ -352,9 +347,9 @@ export const changelogReleases: ChangelogRelease[] = [ href: `${RELEASE_BASE_URL}/v2.0.2`, title: "Agent benchmark, .NET ecosystem, and a 42% lighter install", summary: - "This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the homepage, adds .NET as a first-class ecosystem on the new stack graph, and ships a much leaner install. It also fixes four scaffold bugs the benchmark itself uncovered.", + "This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the benchmark page, adds .NET as a first-class ecosystem on the new stack graph, and ships a much leaner install. It also fixes four scaffold bugs the benchmark itself uncovered.", highlights: [ - "Benchmarked frontier models scaffolding the same project specs three ways - prompt-only, our CLI, and our MCP server. Agents on the MCP path finished up to 7× faster with 4× fewer output tokens; the full results live on the homepage with an interactive chart.", + "Benchmarked frontier models scaffolding the same project specs three ways - prompt-only, our CLI, and our MCP server. Agents on the MCP path finished up to 7× faster with 4× fewer output tokens; the full results were published on the benchmark page at the time, with an interactive chart.", "Redesigned the MCP page with one-paste setup for Claude Code, Codex, Gemini CLI, Cursor, VS Code, Claude Desktop, Windsurf, and Zed.", "Added .NET as a first-class ecosystem, plus an enterprise tier, backend-utils, and Render/Netlify deployment options on the stack graph (Phases 0–4).", "Cut install size by 42% (122 MB → 71 MB) and the web entry chunk by 32%.", diff --git a/apps/web/src/lib/docs/verified-combinations-data.ts b/apps/web/src/lib/docs/verified-combinations-data.ts index f8ef849cd..eb4b37cb0 100644 --- a/apps/web/src/lib/docs/verified-combinations-data.ts +++ b/apps/web/src/lib/docs/verified-combinations-data.ts @@ -25,19 +25,6 @@ export type VerifiedCombinationSummary = { rerunCommand: string; failureHint: string; }>; - scaffbench: Array<{ - label: string; - source: string; - pass: number; - total: number; - current?: boolean; - reasons?: string[]; - environmentQualified?: boolean; - ownerArea: string; - actionLinks: VerifiedCombinationActionLink[]; - rerunCommand: string; - failureHint: string; - }>; releaseGuard: { source: string; pass: number; @@ -70,11 +57,11 @@ export type VerifiedCombinationSummary = { }; export const verifiedCombinationsSummary: VerifiedCombinationSummary = { - "generatedAt": "2026-08-11T11:11:08.988Z", - "expiresAt": "2026-08-12T23:11:08.988Z", - "gitHead": "0c1bc90f735bb15b3fdd7aa1131ec7d48a0f274d", + "generatedAt": "2026-09-03T17:12:53.985Z", + "expiresAt": "2026-09-05T05:12:53.985Z", + "gitHead": "d644ce745a48cad689bb1446789e834a428e3607", "expectedTotals": { - "releaseGuard": 17, + "releaseGuard": 22, "publishedPackage": 3 }, "smoke": [ @@ -104,31 +91,6 @@ export const verifiedCombinationsSummary: VerifiedCombinationSummary = { ] } ], - "scaffbench": [ - { - "label": "ScaffBench 2", - "source": "testing/.tmp-scaffbench-2/summary.json", - "pass": 0, - "total": 1, - "ownerArea": "packages/template-generator/templates", - "actionLinks": [ - { - "label": "runner", - "href": "https://github.com/Marve10s/Better-Fullstack/blob/main/scripts/benchmarks/scaffbench-v2.ts" - }, - { - "label": "owner", - "href": "https://github.com/Marve10s/Better-Fullstack/blob/main/packages/template-generator/templates" - } - ], - "rerunCommand": "bun run scaffbench:2:canonical", - "failureHint": "Inspect failureTags and validation steps in the ScaffBench summary, then follow the owner area for the stack family.", - "current": false, - "reasons": [ - "missing" - ] - } - ], "releaseGuard": null, "publishedPackage": null }; diff --git a/apps/web/src/lib/seo/sitemap-core.ts b/apps/web/src/lib/seo/sitemap-core.ts index 57686a877..3db1aa27e 100644 --- a/apps/web/src/lib/seo/sitemap-core.ts +++ b/apps/web/src/lib/seo/sitemap-core.ts @@ -27,7 +27,6 @@ const staticSitemapEntries: SitemapEntry[] = [ { path: "/compare/create-t3-app", changefreq: "weekly", priority: 0.7 }, { path: "/compare/better-t-stack", changefreq: "weekly", priority: 0.7 }, { path: "/mcp", changefreq: "weekly", priority: 0.7 }, - { path: "/run", changefreq: "weekly", priority: 0.7 }, { path: "/run-before-you-clone", changefreq: "weekly", priority: 0.9 }, { path: "/templates", changefreq: "weekly", priority: 0.9 }, ]; diff --git a/apps/web/src/paraglide/messages/_index.js b/apps/web/src/paraglide/messages/_index.js index a1fe330ff..0b6e652c6 100644 --- a/apps/web/src/paraglide/messages/_index.js +++ b/apps/web/src/paraglide/messages/_index.js @@ -2,11 +2,8 @@ /** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ export * from './actionsrandom1.js' export * from './actionsreset1.js' -export * from './benchmarkseotitle2.js' -export * from './benchmarkteasercta2.js' -export * from './benchmarkteasermcpbody3.js' -export * from './benchmarkteasertitle2.js' -export * from './benchmarkteasertopmodels3.js' +export * from './benchmarkdescription1.js' +export * from './benchmarktitle1.js' export * from './blogallposts2.js' export * from './blogcopylink2.js' export * from './blogdescription1.js' @@ -261,6 +258,47 @@ export * from './docssectiongettingstarted3.js' export * from './docssectionoverview2.js' export * from './docssectionreference2.js' export * from './docstypetosearch3.js' +export * from './fixproofboardcaption2.js' +export * from './fixproofboardheading2.js' +export * from './fixproofchartaxisminutes3.js' +export * from './fixproofchartcaption2.js' +export * from './fixproofchartheading2.js' +export * from './fixproofchartlegendaria3.js' +export * from './fixproofchartmetricaria3.js' +export * from './fixproofchartnote2.js' +export * from './fixproofchartpointaria3.js' +export * from './fixproofchartregionaria3.js' +export * from './fixproofclaim1.js' +export * from './fixproofcolclaimedonly3.js' +export * from './fixproofcoleffort2.js' +export * from './fixproofcolharness2.js' +export * from './fixproofcolmedianminutes3.js' +export * from './fixproofcolmodel2.js' +export * from './fixproofcolprogressindex3.js' +export * from './fixproofcolregressions2.js' +export * from './fixproofcolresolvedindex3.js' +export * from './fixproofcolrundate3.js' +export * from './fixproofcolsolvedovergraded4.js' +export * from './fixproofcoltestedits3.js' +export * from './fixproofcoltrials2.js' +export * from './fixproofdefclaimedonly3.js' +export * from './fixproofdefharness2.js' +export * from './fixproofdefmedianminutes3.js' +export * from './fixproofdefprogressindex3.js' +export * from './fixproofdefregressions2.js' +export * from './fixproofdefresolvedindex3.js' +export * from './fixproofdefsolvedovergraded4.js' +export * from './fixproofdeftestedits3.js' +export * from './fixproofdeftrials2.js' +export * from './fixproofdefinitionaria2.js' +export * from './fixproofgradedoftotal3.js' +export * from './fixproofprovenancesummary2.js' +export * from './fixproofseotitle2.js' +export * from './fixproofsortaria2.js' +export * from './fixproofstatusdatelabel3.js' +export * from './fixproofstatusgradedlabel3.js' +export * from './fixproofstatusrunlabel3.js' +export * from './fixproofstatustrialslabel3.js' export * from './footerbuiltby2.js' export * from './footerchangelog1.js' export * from './footerinspiredby2.js' @@ -336,39 +374,10 @@ export * from './launchradarmodaleyebrow3.js' export * from './launchradarmodaltitle3.js' export * from './launchradaropenbuilder3.js' export * from './launchradaropenunread3.js' -export * from './llmagentdescription2.js' -export * from './llmagenttitle2.js' -export * from './llmallsupportedclients3.js' -export * from './llmavgscaffoldtime3.js' -export * from './llmbenchmarkdescription2.js' -export * from './llmbenchmarkmetric2.js' -export * from './llmbuildspassing2.js' -export * from './llmclaudesweep2.js' -export * from './llmcodexsweep2.js' export * from './llmcopyagentsetupcommand4.js' -export * from './llmerrorrate2.js' -export * from './llmfailedbuilds2.js' -export * from './llmfastreliable2.js' -export * from './llmfiltermodels2.js' -export * from './llmlightsweep2.js' -export * from './llmmodels1.js' -export * from './llmmostefficient2.js' export * from './llmotherclients2.js' -export * from './llmoutputtokens2.js' export * from './llmpasteinto2.js' -export * from './llmpathclidetail3.js' -export * from './llmpathclishort3.js' -export * from './llmpathmcpdetail3.js' -export * from './llmpathpromptdetail3.js' -export * from './llmpathpromptshort3.js' -export * from './llmreadblog2.js' export * from './llmruninterminal3.js' -export * from './llmrunityourself3.js' -export * from './llmscatteraria2.js' -export * from './llmscatterunmetered2.js' -export * from './llmspeed1.js' -export * from './llmtokens1.js' -export * from './llmtrymcp2.js' export * from './mcpcopyagentconfiguration3.js' export * from './mcpdocs1.js' export * from './mcpfinaldescription2.js' @@ -392,7 +401,6 @@ export * from './mcprunterminal2.js' export * from './mcpseodescription2.js' export * from './mcpseotitle2.js' export * from './mcpstatconfigurableoptions3.js' -export * from './mcpstatfasterpromptonly4.js' export * from './mcpstatreadableresources3.js' export * from './mcpstatstructuredtools3.js' export * from './mcpterminalexample2.js' @@ -485,49 +493,6 @@ export * from './presettrackrustbackendname4.js' export * from './presettracksaasdescription3.js' export * from './presettracksaasintent3.js' export * from './presettracksaasname3.js' -export * from './runagentsdesc2.js' -export * from './runagentseyebrow2.js' -export * from './runagentstitle2.js' -export * from './runauthapidesc3.js' -export * from './runauthapitab3.js' -export * from './runauthclidesc3.js' -export * from './runauthclitab3.js' -export * from './runcolagent2.js' -export * from './runcolauth2.js' -export * from './runcolmodels2.js' -export * from './runctadesc2.js' -export * from './runctaeyebrow2.js' -export * from './runctaleaderboard2.js' -export * from './runctatitle2.js' -export * from './runflagefforts2.js' -export * from './runflagmodel2.js' -export * from './runflagoutdir3.js' -export * from './runflagpaths2.js' -export * from './runflagphase2.js' -export * from './runflagspecs2.js' -export * from './runflagseyebrow2.js' -export * from './runflagstitle2.js' -export * from './runherobrowsereports3.js' -export * from './runherodescription2.js' -export * from './runheroeyebrow2.js' -export * from './runheroquickstart2.js' -export * from './runherotitlea3.js' -export * from './runherotitleb3.js' -export * from './runlabelclone2.js' -export * from './runlabelexportkey3.js' -export * from './runlabelrunall3.js' -export * from './runlabelsignin2.js' -export * from './runlabeltwophase3.js' -export * from './runquickstarteyebrow2.js' -export * from './runquickstarttitle2.js' -export * from './runresultsnotelink3.js' -export * from './runresultsnotepre3.js' -export * from './runseodescription2.js' -export * from './runseotitle2.js' -export * from './runstepauth2.js' -export * from './runstepcloneinstall3.js' -export * from './runsteprun2.js' -export * from './runtwophasenote3.js' export * from './saveddelete1.js' export * from './saveddeletedescription2.js' export * from './saveddeletepreset2.js' diff --git a/apps/web/src/paraglide/messages/benchmarkdescription1.js b/apps/web/src/paraglide/messages/benchmarkdescription1.js new file mode 100644 index 000000000..7c5778d57 --- /dev/null +++ b/apps/web/src/paraglide/messages/benchmarkdescription1.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Benchmarkdescription1Inputs */ + +const en_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof grades coding agents on sealed, real issues from private and public codebases, verified by hidden tests.`) +}; + +const es_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof evalúa agentes de programación con errores reales y sellados de bases de código privadas y públicas, verificados por pruebas ocultas.`) +}; + +const zh_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof 用来自私有和公开代码库的封闭真实问题评测编程代理,并由隐藏测试验证。`) +}; + +const ja_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof は、非公開および公開コードベースから集めた封印済みの実際の不具合でコーディングエージェントを採点し、非公開テストで検証します。`) +}; + +const ko_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof는 비공개 및 공개 코드베이스에서 가져온 봉인된 실제 이슈로 코딩 에이전트를 채점하고, 비공개 테스트로 검증합니다.`) +}; + +const zh_hant1_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof 用來自私有和公開程式碼庫的封閉真實問題評測程式代理程式,並由隱藏測試驗證。`) +}; + +const de_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof bewertet Coding-Agenten an versiegelten, echten Fehlern aus privaten und öffentlichen Codebasen, verifiziert durch verborgene Tests.`) +}; + +const fr_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof évalue les agents de codage sur des bugs réels et scellés issus de bases de code privées et publiques, vérifiés par des tests cachés.`) +}; + +const uk_benchmarkdescription1 = /** @type {(inputs: Benchmarkdescription1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof оцінює агентів для коду на закритих реальних помилках із приватних і публічних кодових баз, перевірених прихованими тестами.`) +}; + +/** +* | output | +* | --- | +* | "Fixproof grades coding agents on sealed, real issues from private and public codebases, verified by hidden tests." | +* +* @param {Benchmarkdescription1Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const benchmarkdescription1 = /** @type {((inputs?: Benchmarkdescription1Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_benchmarkdescription1(inputs) + if (locale === "zh") return zh_benchmarkdescription1(inputs) + if (locale === "ja") return ja_benchmarkdescription1(inputs) + if (locale === "ko") return ko_benchmarkdescription1(inputs) + if (locale === "zh-Hant") return zh_hant1_benchmarkdescription1(inputs) + if (locale === "de") return de_benchmarkdescription1(inputs) + if (locale === "fr") return fr_benchmarkdescription1(inputs) + if (locale === "uk") return uk_benchmarkdescription1(inputs) + return en_benchmarkdescription1(inputs) +}); +export { benchmarkdescription1 as "benchmarkDescription" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/benchmarkseotitle2.js b/apps/web/src/paraglide/messages/benchmarkseotitle2.js deleted file mode 100644 index 633609085..000000000 --- a/apps/web/src/paraglide/messages/benchmarkseotitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Benchmarkseotitle2Inputs */ - -const en_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - How good are AI models at building your projects?`) -}; - -const es_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - ¿Qué tan buenos son los modelos de IA construyendo tus proyectos?`) -}; - -const zh_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - AI 模型构建你的项目有多强?`) -}; - -const ja_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - AIモデルはあなたのプロジェクトをどれだけうまく構築できるか?`) -}; - -const ko_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - AI 모델은 당신의 프로젝트를 얼마나 잘 만들까요?`) -}; - -const zh_hant1_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - AI 模型建構你的專案有多強?`) -}; - -const de_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - Wie gut bauen KI-Modelle deine Projekte?`) -}; - -const fr_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - Les modèles d'IA sont-ils vraiment bons pour créer vos projets ?`) -}; - -const uk_benchmarkseotitle2 = /** @type {(inputs: Benchmarkseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench - Наскільки добре моделі ШІ створюють ваші проєкти?`) -}; - -/** -* | output | -* | --- | -* | "ScaffBench - How good are AI models at building your projects?" | -* -* @param {Benchmarkseotitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const benchmarkseotitle2 = /** @type {((inputs?: Benchmarkseotitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_benchmarkseotitle2(inputs) - if (locale === "zh") return zh_benchmarkseotitle2(inputs) - if (locale === "ja") return ja_benchmarkseotitle2(inputs) - if (locale === "ko") return ko_benchmarkseotitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_benchmarkseotitle2(inputs) - if (locale === "de") return de_benchmarkseotitle2(inputs) - if (locale === "fr") return fr_benchmarkseotitle2(inputs) - if (locale === "uk") return uk_benchmarkseotitle2(inputs) - return en_benchmarkseotitle2(inputs) -}); -export { benchmarkseotitle2 as "benchmarkSeoTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/benchmarkteasercta2.js b/apps/web/src/paraglide/messages/benchmarkteasercta2.js deleted file mode 100644 index 582f3ef51..000000000 --- a/apps/web/src/paraglide/messages/benchmarkteasercta2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Benchmarkteasercta2Inputs */ - -const en_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`See the full benchmark`) -}; - -const es_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ver el benchmark completo`) -}; - -const zh_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`查看完整基准测试`) -}; - -const ja_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ベンチマーク全体を見る`) -}; - -const ko_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`전체 벤치마크 보기`) -}; - -const zh_hant1_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`查看完整基準測試`) -}; - -const de_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Vollständigen Benchmark ansehen`) -}; - -const fr_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Voir le benchmark complet`) -}; - -const uk_benchmarkteasercta2 = /** @type {(inputs: Benchmarkteasercta2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Переглянути повний бенчмарк`) -}; - -/** -* | output | -* | --- | -* | "See the full benchmark" | -* -* @param {Benchmarkteasercta2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const benchmarkteasercta2 = /** @type {((inputs?: Benchmarkteasercta2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_benchmarkteasercta2(inputs) - if (locale === "zh") return zh_benchmarkteasercta2(inputs) - if (locale === "ja") return ja_benchmarkteasercta2(inputs) - if (locale === "ko") return ko_benchmarkteasercta2(inputs) - if (locale === "zh-Hant") return zh_hant1_benchmarkteasercta2(inputs) - if (locale === "de") return de_benchmarkteasercta2(inputs) - if (locale === "fr") return fr_benchmarkteasercta2(inputs) - if (locale === "uk") return uk_benchmarkteasercta2(inputs) - return en_benchmarkteasercta2(inputs) -}); -export { benchmarkteasercta2 as "benchmarkTeaserCta" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/benchmarkteasermcpbody3.js b/apps/web/src/paraglide/messages/benchmarkteasermcpbody3.js deleted file mode 100644 index f8ed49f50..000000000 --- a/apps/web/src/paraglide/messages/benchmarkteasermcpbody3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Benchmarkteasermcpbody3Inputs */ - -const en_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`The difference is our MCP. Point any coding agent at Better-Fullstack's tools and even a small free model builds almost everything - with a fraction of the tokens and steps.`) -}; - -const es_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`La diferencia es nuestro MCP. Conecta cualquier agente de programación a las herramientas de Better-Fullstack y hasta un pequeño modelo gratuito construye casi todo, con una fracción de los tokens y los pasos.`) -}; - -const zh_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`差别在于我们的 MCP。让任何编程代理接入 Better-Fullstack 的工具,即使是小型免费模型也能用极少的 tokens 和步骤构建几乎一切。`) -}; - -const ja_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`違いは私たちのMCPです。あらゆるコーディングエージェントをBetter-Fullstackのツールに向ければ、小さな無料モデルでもわずかなトークンとステップでほぼすべてを構築します。`) -}; - -const ko_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`차이는 우리의 MCP입니다. 어떤 코딩 에이전트든 Better-Fullstack 도구에 연결하면 작은 무료 모델조차 훨씬 적은 토큰과 단계로 거의 모든 것을 만들어 냅니다.`) -}; - -const zh_hant1_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`差別在於我們的 MCP。讓任何程式設計代理接入 Better-Fullstack 的工具,即使是小型免費模型也能用極少的 tokens 和步驟建構幾乎一切。`) -}; - -const de_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Der Unterschied ist unser MCP. Richte einen beliebigen Coding-Agenten auf die Tools von Better-Fullstack und selbst ein kleines kostenloses Modell baut fast alles – mit einem Bruchteil der Tokens und Schritte.`) -}; - -const fr_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`La différence, c'est notre MCP. Connectez n'importe quel agent de codage aux outils de Better-Fullstack et même un petit modèle gratuit construit presque tout, avec une fraction des jetons et des étapes.`) -}; - -const uk_benchmarkteasermcpbody3 = /** @type {(inputs: Benchmarkteasermcpbody3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Різниця - у нашому MCP. Спрямуйте будь-якого агента для коду на інструменти Better-Fullstack, і навіть маленька безкоштовна модель збудує майже все - з часткою токенів і кроків.`) -}; - -/** -* | output | -* | --- | -* | "The difference is our MCP. Point any coding agent at Better-Fullstack's tools and even a small free model builds almost everything - with a fraction of the t..." | -* -* @param {Benchmarkteasermcpbody3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const benchmarkteasermcpbody3 = /** @type {((inputs?: Benchmarkteasermcpbody3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_benchmarkteasermcpbody3(inputs) - if (locale === "zh") return zh_benchmarkteasermcpbody3(inputs) - if (locale === "ja") return ja_benchmarkteasermcpbody3(inputs) - if (locale === "ko") return ko_benchmarkteasermcpbody3(inputs) - if (locale === "zh-Hant") return zh_hant1_benchmarkteasermcpbody3(inputs) - if (locale === "de") return de_benchmarkteasermcpbody3(inputs) - if (locale === "fr") return fr_benchmarkteasermcpbody3(inputs) - if (locale === "uk") return uk_benchmarkteasermcpbody3(inputs) - return en_benchmarkteasermcpbody3(inputs) -}); -export { benchmarkteasermcpbody3 as "benchmarkTeaserMcpBody" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/benchmarkteasertitle2.js b/apps/web/src/paraglide/messages/benchmarkteasertitle2.js deleted file mode 100644 index e408fb51d..000000000 --- a/apps/web/src/paraglide/messages/benchmarkteasertitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Benchmarkteasertitle2Inputs */ - -const en_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`How good are AI models at building your projects?`) -}; - -const es_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`¿Qué tan buenos son los modelos de IA construyendo tus proyectos?`) -}; - -const zh_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`AI 模型构建你的项目有多强?`) -}; - -const ja_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`AIモデルはあなたのプロジェクトをどれだけうまく構築できるか?`) -}; - -const ko_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`AI 모델은 당신의 프로젝트를 얼마나 잘 만들까요?`) -}; - -const zh_hant1_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`AI 模型建構你的專案有多強?`) -}; - -const de_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Wie gut bauen KI-Modelle deine Projekte?`) -}; - -const fr_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Les modèles d'IA sont-ils vraiment bons pour créer vos projets ?`) -}; - -const uk_benchmarkteasertitle2 = /** @type {(inputs: Benchmarkteasertitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Наскільки добре моделі ШІ створюють ваші проєкти?`) -}; - -/** -* | output | -* | --- | -* | "How good are AI models at building your projects?" | -* -* @param {Benchmarkteasertitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const benchmarkteasertitle2 = /** @type {((inputs?: Benchmarkteasertitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_benchmarkteasertitle2(inputs) - if (locale === "zh") return zh_benchmarkteasertitle2(inputs) - if (locale === "ja") return ja_benchmarkteasertitle2(inputs) - if (locale === "ko") return ko_benchmarkteasertitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_benchmarkteasertitle2(inputs) - if (locale === "de") return de_benchmarkteasertitle2(inputs) - if (locale === "fr") return fr_benchmarkteasertitle2(inputs) - if (locale === "uk") return uk_benchmarkteasertitle2(inputs) - return en_benchmarkteasertitle2(inputs) -}); -export { benchmarkteasertitle2 as "benchmarkTeaserTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/benchmarkteasertopmodels3.js b/apps/web/src/paraglide/messages/benchmarkteasertopmodels3.js deleted file mode 100644 index d33ab209b..000000000 --- a/apps/web/src/paraglide/messages/benchmarkteasertopmodels3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Benchmarkteasertopmodels3Inputs */ - -const en_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Top models`) -}; - -const es_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Mejores modelos`) -}; - -const zh_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`顶尖模型`) -}; - -const ja_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`トップモデル`) -}; - -const ko_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`상위 모델`) -}; - -const zh_hant1_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`頂尖模型`) -}; - -const de_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Top-Modelle`) -}; - -const fr_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Meilleurs modèles`) -}; - -const uk_benchmarkteasertopmodels3 = /** @type {(inputs: Benchmarkteasertopmodels3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Найкращі моделі`) -}; - -/** -* | output | -* | --- | -* | "Top models" | -* -* @param {Benchmarkteasertopmodels3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const benchmarkteasertopmodels3 = /** @type {((inputs?: Benchmarkteasertopmodels3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_benchmarkteasertopmodels3(inputs) - if (locale === "zh") return zh_benchmarkteasertopmodels3(inputs) - if (locale === "ja") return ja_benchmarkteasertopmodels3(inputs) - if (locale === "ko") return ko_benchmarkteasertopmodels3(inputs) - if (locale === "zh-Hant") return zh_hant1_benchmarkteasertopmodels3(inputs) - if (locale === "de") return de_benchmarkteasertopmodels3(inputs) - if (locale === "fr") return fr_benchmarkteasertopmodels3(inputs) - if (locale === "uk") return uk_benchmarkteasertopmodels3(inputs) - return en_benchmarkteasertopmodels3(inputs) -}); -export { benchmarkteasertopmodels3 as "benchmarkTeaserTopModels" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/benchmarktitle1.js b/apps/web/src/paraglide/messages/benchmarktitle1.js new file mode 100644 index 000000000..4a1e21495 --- /dev/null +++ b/apps/web/src/paraglide/messages/benchmarktitle1.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Benchmarktitle1Inputs */ + +const en_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const es_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const zh_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const ja_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const ko_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const zh_hant1_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const de_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const fr_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +const uk_benchmarktitle1 = /** @type {(inputs: Benchmarktitle1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof`) +}; + +/** +* | output | +* | --- | +* | "Fixproof" | +* +* @param {Benchmarktitle1Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const benchmarktitle1 = /** @type {((inputs?: Benchmarktitle1Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_benchmarktitle1(inputs) + if (locale === "zh") return zh_benchmarktitle1(inputs) + if (locale === "ja") return ja_benchmarktitle1(inputs) + if (locale === "ko") return ko_benchmarktitle1(inputs) + if (locale === "zh-Hant") return zh_hant1_benchmarktitle1(inputs) + if (locale === "de") return de_benchmarktitle1(inputs) + if (locale === "fr") return fr_benchmarktitle1(inputs) + if (locale === "uk") return uk_benchmarktitle1(inputs) + return en_benchmarktitle1(inputs) +}); +export { benchmarktitle1 as "benchmarkTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/changelogrelease20260612highlightbenchmark3.js b/apps/web/src/paraglide/messages/changelogrelease20260612highlightbenchmark3.js index 79d00c046..f46f3d14c 100644 --- a/apps/web/src/paraglide/messages/changelogrelease20260612highlightbenchmark3.js +++ b/apps/web/src/paraglide/messages/changelogrelease20260612highlightbenchmark3.js @@ -6,39 +6,39 @@ import { getLocale, experimentalStaticLocale } from '../runtime.js'; /** @typedef {{}} Changelogrelease20260612highlightbenchmark3Inputs */ const en_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmarked frontier models scaffolding the same project specs three ways: prompt-only, our CLI, and our MCP server. Agents on the MCP path finished up to 7× faster with 4× fewer output tokens; the full results live on the homepage with an interactive chart.`) + return /** @type {LocalizedString} */ (`Benchmarked frontier models scaffolding the same project specs three ways: prompt-only, our CLI, and our MCP server. Agents on the MCP path finished up to 7× faster with 4× fewer output tokens; the full results were published on the benchmark page at the time, with an interactive chart.`) }; const es_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Se probaron modelos de frontera creando los mismos specs de proyecto por tres rutas: solo prompt, nuestra CLI y nuestro servidor MCP. En la ruta MCP, los agentes terminaron hasta 7× más rápido con 4× menos tokens de salida; los resultados completos están en la página de inicio con un gráfico interactivo.`) + return /** @type {LocalizedString} */ (`Se probaron modelos de frontera creando los mismos specs de proyecto por tres rutas: solo prompt, nuestra CLI y nuestro servidor MCP. En la ruta MCP, los agentes terminaron hasta 7× más rápido con 4× menos tokens de salida; los resultados completos se publicaron entonces en la página del benchmark, con un gráfico interactivo.`) }; const zh_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`用同一组项目 spec 测试前沿模型的三种 scaffold 路径:纯 prompt、我们的 CLI、以及我们的 MCP 服务器。走 MCP 路径的代理最高快 7×,输出 tokens 少 4×;完整结果已在首页通过交互图展示。`) + return /** @type {LocalizedString} */ (`用同一组项目 spec 测试前沿模型的三种 scaffold 路径:纯 prompt、我们的 CLI、以及我们的 MCP 服务器。走 MCP 路径的代理最高快 7×,输出 tokens 少 4×;完整结果当时已在 benchmark 页面通过交互图展示。`) }; const ja_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ベンチマークされたフロンティア モデルは、同じプロジェクト仕様を 3 つの方法 (プロンプトのみ、CLI、MCP サーバー) でスキャフォールディングします。MCP パス上のエージェントは、出力トークンが 4 分の 1 で、最大 7 倍の速さで完了しました。完全な結果は、インタラクティブなグラフとともにホームページに表示されます。`) + return /** @type {LocalizedString} */ (`ベンチマークされたフロンティア モデルは、同じプロジェクト仕様を 3 つの方法 (プロンプトのみ、CLI、MCP サーバー) でスキャフォールディングします。MCP パス上のエージェントは、出力トークンが 4 分の 1 で、最大 7 倍の速さで完了しました。完全な結果は当時、インタラクティブなグラフとともにベンチマークページで公開されました。`) }; const ko_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`동일한 프로젝트 사양을 세 가지 방식으로 스캐폴딩하는 벤치마킹된 프론티어 모델: 프롬프트 전용, CLI 및 MCP 서버. MCP 경로의 에이전트는 4배 더 적은 출력 토큰으로 최대 7배 더 빠르게 완료되었습니다. 전체 결과는 대화형 차트와 함께 홈페이지에 게시됩니다.`) + return /** @type {LocalizedString} */ (`동일한 프로젝트 사양을 세 가지 방식으로 스캐폴딩하는 벤치마킹된 프론티어 모델: 프롬프트 전용, CLI 및 MCP 서버. MCP 경로의 에이전트는 4배 더 적은 출력 토큰으로 최대 7배 더 빠르게 완료되었습니다. 전체 결과는 당시 대화형 차트와 함께 벤치마크 페이지에 게시되었습니다.`) }; const zh_hant1_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`用同一組專案 spec 測試前沿模型的三種 scaffold 路徑:純 prompt、我們的 CLI、以及我們的 MCP 伺服器。走 MCP 路徑的代理程式最高快 7×,輸出 tokens 少 4×;完整結果已在首頁透過互動式圖表展示。`) + return /** @type {LocalizedString} */ (`用同一組專案 spec 測試前沿模型的三種 scaffold 路徑:純 prompt、我們的 CLI、以及我們的 MCP 伺服器。走 MCP 路徑的代理程式最高快 7×,輸出 tokens 少 4×;完整結果當時已在 benchmark 頁面透過互動式圖表展示。`) }; const de_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Wir haben Frontier-Modelle beim Erstellen derselben Projektspezifikationen auf drei Wegen verglichen: nur per Prompt, über unser CLI und über unseren MCP-Server. Agenten auf dem MCP-Pfad waren bis zu 7× schneller bei 4× weniger Ausgabetokens; die vollständigen Ergebnisse gibt es live auf der Homepage mit einem interaktiven Diagramm.`) + return /** @type {LocalizedString} */ (`Wir haben Frontier-Modelle beim Erstellen derselben Projektspezifikationen auf drei Wegen verglichen: nur per Prompt, über unser CLI und über unseren MCP-Server. Agenten auf dem MCP-Pfad waren bis zu 7× schneller bei 4× weniger Ausgabetokens; die vollständigen Ergebnisse wurden damals mit einem interaktiven Diagramm auf der Benchmark-Seite veröffentlicht.`) }; const fr_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Nous avons comparé des modèles de pointe qui échafaudent les mêmes spécifications de projet de trois manières : par invite uniquement, par notre CLI et par notre serveur MCP. Les agents sur le chemin MCP ont terminé jusqu'à 7 fois plus vite avec 4 fois moins de jetons de sortie ; les résultats complets sont disponibles sur la page d'accueil avec un graphique interactif.`) + return /** @type {LocalizedString} */ (`Nous avons comparé des modèles de pointe qui échafaudent les mêmes spécifications de projet de trois manières : par invite uniquement, par notre CLI et par notre serveur MCP. Les agents sur le chemin MCP ont terminé jusqu'à 7 fois plus vite avec 4 fois moins de jetons de sortie ; les résultats complets ont été publiés à l'époque sur la page du benchmark, avec un graphique interactif.`) }; const uk_changelogrelease20260612highlightbenchmark3 = /** @type {(inputs: Changelogrelease20260612highlightbenchmark3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Порівняли frontier-моделі, які генерують той самий проєкт трьома шляхами: лише prompt, наш CLI і MCP-сервер. На MCP-шляху агенти завершували до 7× швидше з 4× меншою кількістю output-токенів; повні результати є на головній з інтерактивним графіком.`) + return /** @type {LocalizedString} */ (`Порівняли frontier-моделі, які генерують той самий проєкт трьома шляхами: лише prompt, наш CLI і MCP-сервер. На MCP-шляху агенти завершували до 7× швидше з 4× меншою кількістю output-токенів; повні результати тоді опублікували на сторінці бенчмарка з інтерактивним графіком.`) }; /** diff --git a/apps/web/src/paraglide/messages/changelogrelease20260612summary2.js b/apps/web/src/paraglide/messages/changelogrelease20260612summary2.js index 4b14c6b58..ed82476f0 100644 --- a/apps/web/src/paraglide/messages/changelogrelease20260612summary2.js +++ b/apps/web/src/paraglide/messages/changelogrelease20260612summary2.js @@ -6,45 +6,45 @@ import { getLocale, experimentalStaticLocale } from '../runtime.js'; /** @typedef {{}} Changelogrelease20260612summary2Inputs */ const en_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the homepage, adds .NET as a first-class ecosystem on the new stack graph, and ships a much leaner install. It also fixes four scaffold bugs the benchmark itself uncovered.`) + return /** @type {LocalizedString} */ (`This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the benchmark page, adds .NET as a first-class ecosystem on the new stack graph, and ships a much leaner install. It also fixes four scaffold bugs the benchmark itself uncovered.`) }; const es_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Esta versión mide cómo los agentes de IA crean scaffolds con Better Fullstack y publica los resultados en la página de inicio, añade .NET como ecosistema de primera clase en el nuevo grafo de stacks y entrega una instalación mucho más ligera. También corrige cuatro errores de scaffold que descubrió el propio benchmark.`) + return /** @type {LocalizedString} */ (`Esta versión mide cómo los agentes de IA crean scaffolds con Better Fullstack y publica los resultados en la página del benchmark, añade .NET como ecosistema de primera clase en el nuevo grafo de stacks y entrega una instalación mucho más ligera. También corrige cuatro errores de scaffold que descubrió el propio benchmark.`) }; const zh_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`这个版本衡量 AI 代理如何使用 Better Fullstack 生成 scaffold,并把结果发布到首页;同时在新的 stack graph 中加入一等 .NET 生态,并带来更轻的安装包。它还修复了 benchmark 本身发现的四个 scaffold 问题。`) + return /** @type {LocalizedString} */ (`这个版本衡量 AI 代理如何使用 Better Fullstack 生成 scaffold,并把结果发布到 benchmark 页面;同时在新的 stack graph 中加入一等 .NET 生态,并带来更轻的安装包。它还修复了 benchmark 本身发现的四个 scaffold 问题。`) }; const ja_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`このリリースでは、AI エージェントが Better Fullstack でどのようにスキャフォールディングするかをベンチマークし、その結果をホームページで公開し、新しいスタック グラフにファーストクラスのエコシステムとして .NET を追加し、より無駄のないインストールをリリースします。また、ベンチマーク自体が発見した 4 つのスキャフォールドのバグも修正されています。`) + return /** @type {LocalizedString} */ (`このリリースでは、AI エージェントが Better Fullstack でどのようにスキャフォールディングするかをベンチマークし、その結果をベンチマークページで公開し、新しいスタック グラフにファーストクラスのエコシステムとして .NET を追加し、より無駄のないインストールをリリースします。また、ベンチマーク自体が発見した 4 つのスキャフォールドのバグも修正されています。`) }; const ko_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`이 릴리스에서는 AI 에이전트가 Better Fullstack을 사용하여 스캐폴드하고 결과를 홈페이지에 게시하는 방법을 벤치마킹하고, 새 스택 그래프에 .NET을 일류 생태계로 추가하고, 훨씬 더 간결한 설치를 제공합니다. 또한 벤치마크 자체에서 발견한 4가지 스캐폴드 버그도 수정합니다.`) + return /** @type {LocalizedString} */ (`이 릴리스에서는 AI 에이전트가 Better Fullstack을 사용하여 스캐폴드하고 결과를 벤치마크 페이지에 게시하는 방법을 벤치마킹하고, 새 스택 그래프에 .NET을 일류 생태계로 추가하고, 훨씬 더 간결한 설치를 제공합니다. 또한 벤치마크 자체에서 발견한 4가지 스캐폴드 버그도 수정합니다.`) }; const zh_hant1_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`這個版本衡量 AI 代理程式如何使用 Better Fullstack 產生 scaffold,並將結果發佈到首頁;同時在新的 stack graph 中加入一等 .NET 生態,並帶來更輕的安裝包。它還修復了 benchmark 本身發現的四個 scaffold 問題。`) + return /** @type {LocalizedString} */ (`這個版本衡量 AI 代理程式如何使用 Better Fullstack 產生 scaffold,並將結果發佈到 benchmark 頁面;同時在新的 stack graph 中加入一等 .NET 生態,並帶來更輕的安裝包。它還修復了 benchmark 本身發現的四個 scaffold 問題。`) }; const de_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Diese Version misst, wie AI-Agenten mit Better Fullstack ein Gerüst bilden, veröffentlicht die Ergebnisse auf der Homepage, fügt .NET als erstklassiges Ökosystem zum neuen Stapeldiagramm hinzu und liefert eine viel schlankere Installation. Es behebt außerdem vier Gerüstfehler, die der Benchmark selbst aufgedeckt hat.`) + return /** @type {LocalizedString} */ (`Diese Version misst, wie AI-Agenten mit Better Fullstack ein Gerüst bilden, veröffentlicht die Ergebnisse auf der Benchmark-Seite, fügt .NET als erstklassiges Ökosystem zum neuen Stapeldiagramm hinzu und liefert eine viel schlankere Installation. Es behebt außerdem vier Gerüstfehler, die der Benchmark selbst aufgedeckt hat.`) }; const fr_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Cette version évalue la façon dont les agents AI échafaudent avec Better Fullstack et publie les résultats sur la page d'accueil, ajoute .NET en tant qu'écosystème de première classe sur le nouveau graphique de pile et fournit une installation beaucoup plus légère. Elle corrige également quatre bugs d’échafaudage découverts par le benchmark lui-même.`) + return /** @type {LocalizedString} */ (`Cette version évalue la façon dont les agents AI échafaudent avec Better Fullstack et publie les résultats sur la page du benchmark, ajoute .NET en tant qu'écosystème de première classe sur le nouveau graphique de pile et fournit une installation beaucoup plus légère. Elle corrige également quatre bugs d’échafaudage découverts par le benchmark lui-même.`) }; const uk_changelogrelease20260612summary2 = /** @type {(inputs: Changelogrelease20260612summary2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`У цьому релізі ми виміряли, як AI-агенти генерують проєкти з Better Fullstack, опублікували результати на головній, додали .NET як повноцінну екосистему в граф стеку й суттєво полегшили встановлення. Також виправили чотири помилки скафолдингу, які сам бенчмарк і виявив.`) + return /** @type {LocalizedString} */ (`У цьому релізі ми виміряли, як AI-агенти генерують проєкти з Better Fullstack, опублікували результати на сторінці бенчмарка, додали .NET як повноцінну екосистему в граф стеку й суттєво полегшили встановлення. Також виправили чотири помилки скафолдингу, які сам бенчмарк і виявив.`) }; /** * | output | * | --- | -* | "This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the homepage, adds .NET as a first-class ecosystem on the n..." | +* | "This release benchmarks how AI agents scaffold with Better Fullstack and publishes the results on the benchmark page, adds .NET as a first-class ecosystem on..." | * * @param {Changelogrelease20260612summary2Inputs} inputs * @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options diff --git a/apps/web/src/paraglide/messages/fixproofboardcaption2.js b/apps/web/src/paraglide/messages/fixproofboardcaption2.js new file mode 100644 index 000000000..f36a787dc --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofboardcaption2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofboardcaption2Inputs */ + +const en_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`One row per model. Sort by either index. The question mark on a column explains what it counts.`) +}; + +const es_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Una fila por modelo. Ordena por cualquiera de los dos índices. El signo de interrogación de cada columna explica qué cuenta.`) +}; + +const zh_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每个模型一行。可按任一指数排序。列上的问号会说明这一列统计的是什么。`) +}; + +const ja_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`1 行が 1 モデルです。どちらの指数でも並べ替えられます。列の疑問符は、その列が何を数えているかを説明します。`) +}; + +const ko_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`모델마다 한 행입니다. 두 지수 중 어느 쪽으로도 정렬할 수 있습니다. 열의 물음표는 그 열이 무엇을 세는지 설명합니다.`) +}; + +const zh_hant1_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每個模型一列。可按任一指數排序。欄位上的問號會說明這一欄統計的是什麼。`) +}; + +const de_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Eine Zeile pro Modell. Sortierbar nach beiden Indizes. Das Fragezeichen an einer Spalte erklärt, was sie zählt.`) +}; + +const fr_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Une ligne par modèle. Triez selon l'un ou l'autre indice. Le point d'interrogation d'une colonne explique ce qu'elle compte.`) +}; + +const uk_fixproofboardcaption2 = /** @type {(inputs: Fixproofboardcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Один рядок на модель. Сортування за будь-яким з індексів. Знак питання біля колонки пояснює, що вона рахує.`) +}; + +/** +* | output | +* | --- | +* | "One row per model. Sort by either index. The question mark on a column explains what it counts." | +* +* @param {Fixproofboardcaption2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofboardcaption2 = /** @type {((inputs?: Fixproofboardcaption2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofboardcaption2(inputs) + if (locale === "zh") return zh_fixproofboardcaption2(inputs) + if (locale === "ja") return ja_fixproofboardcaption2(inputs) + if (locale === "ko") return ko_fixproofboardcaption2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofboardcaption2(inputs) + if (locale === "de") return de_fixproofboardcaption2(inputs) + if (locale === "fr") return fr_fixproofboardcaption2(inputs) + if (locale === "uk") return uk_fixproofboardcaption2(inputs) + return en_fixproofboardcaption2(inputs) +}); +export { fixproofboardcaption2 as "fixproofBoardCaption" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofboardheading2.js b/apps/web/src/paraglide/messages/fixproofboardheading2.js new file mode 100644 index 000000000..758ab06d7 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofboardheading2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofboardheading2Inputs */ + +const en_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Board`) +}; + +const es_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tabla`) +}; + +const zh_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`榜单`) +}; + +const ja_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`ボード`) +}; + +const ko_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`보드`) +}; + +const zh_hant1_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`榜單`) +}; + +const de_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Rangliste`) +}; + +const fr_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tableau`) +}; + +const uk_fixproofboardheading2 = /** @type {(inputs: Fixproofboardheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Таблиця`) +}; + +/** +* | output | +* | --- | +* | "Board" | +* +* @param {Fixproofboardheading2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofboardheading2 = /** @type {((inputs?: Fixproofboardheading2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofboardheading2(inputs) + if (locale === "zh") return zh_fixproofboardheading2(inputs) + if (locale === "ja") return ja_fixproofboardheading2(inputs) + if (locale === "ko") return ko_fixproofboardheading2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofboardheading2(inputs) + if (locale === "de") return de_fixproofboardheading2(inputs) + if (locale === "fr") return fr_fixproofboardheading2(inputs) + if (locale === "uk") return uk_fixproofboardheading2(inputs) + return en_fixproofboardheading2(inputs) +}); +export { fixproofboardheading2 as "fixproofBoardHeading" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartaxisminutes3.js b/apps/web/src/paraglide/messages/fixproofchartaxisminutes3.js new file mode 100644 index 000000000..5e1e7b7ee --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartaxisminutes3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofchartaxisminutes3Inputs */ + +const en_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Median agent minutes per task`) +}; + +const es_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Mediana de minutos del agente por tarea`) +}; + +const zh_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每个任务的 agent 耗时中位数(分钟)`) +}; + +const ja_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`タスクあたりのエージェント所要時間の中央値 (分)`) +}; + +const ko_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`태스크당 에이전트 소요 시간 중앙값 (분)`) +}; + +const zh_hant1_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每個任務的 agent 耗時中位數(分鐘)`) +}; + +const de_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Median-Agentminuten pro Aufgabe`) +}; + +const fr_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Minutes médianes de l'agent par tâche`) +}; + +const uk_fixproofchartaxisminutes3 = /** @type {(inputs: Fixproofchartaxisminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Медіана хвилин роботи агента на задачу`) +}; + +/** +* | output | +* | --- | +* | "Median agent minutes per task" | +* +* @param {Fixproofchartaxisminutes3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartaxisminutes3 = /** @type {((inputs?: Fixproofchartaxisminutes3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartaxisminutes3(inputs) + if (locale === "zh") return zh_fixproofchartaxisminutes3(inputs) + if (locale === "ja") return ja_fixproofchartaxisminutes3(inputs) + if (locale === "ko") return ko_fixproofchartaxisminutes3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartaxisminutes3(inputs) + if (locale === "de") return de_fixproofchartaxisminutes3(inputs) + if (locale === "fr") return fr_fixproofchartaxisminutes3(inputs) + if (locale === "uk") return uk_fixproofchartaxisminutes3(inputs) + return en_fixproofchartaxisminutes3(inputs) +}); +export { fixproofchartaxisminutes3 as "fixproofChartAxisMinutes" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartcaption2.js b/apps/web/src/paraglide/messages/fixproofchartcaption2.js new file mode 100644 index 000000000..9a1b14fe6 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartcaption2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofchartcaption2Inputs */ + +const en_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`One point per model and effort. Minutes run from slow on the left to fast on the right, so the strongest runs sit toward the top right.`) +}; + +const es_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Un punto por modelo y esfuerzo. Los minutos van de lento a la izquierda a rápido a la derecha, así que las mejores ejecuciones quedan arriba a la derecha.`) +}; + +const zh_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每个模型与推理强度组合一个点。横轴左慢右快,因此表现最好的运行位于右上角。`) +}; + +const ja_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`モデルと推論強度の組み合わせごとに 1 点です。横軸は左が遅く右が速いので、優れた実行ほど右上に寄ります。`) +}; + +const ko_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`모델과 추론 강도 조합마다 점 하나입니다. 가로축은 왼쪽이 느리고 오른쪽이 빠르므로 좋은 실행일수록 오른쪽 위에 놓입니다.`) +}; + +const zh_hant1_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每個模型與推理強度組合一個點。橫軸左慢右快,因此表現最好的執行位於右上角。`) +}; + +const de_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Ein Punkt pro Modell und Effort. Die Minuten laufen von langsam links zu schnell rechts, die stärksten Läufe liegen also oben rechts.`) +}; + +const fr_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Un point par modèle et par effort. Les minutes vont de lent à gauche à rapide à droite, donc les meilleures exécutions se placent en haut à droite.`) +}; + +const uk_fixproofchartcaption2 = /** @type {(inputs: Fixproofchartcaption2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Одна точка на модель і рівень зусиль. Хвилини йдуть від повільних ліворуч до швидких праворуч, тож найкращі запуски опиняються вгорі праворуч.`) +}; + +/** +* | output | +* | --- | +* | "One point per model and effort. Minutes run from slow on the left to fast on the right, so the strongest runs sit toward the top right." | +* +* @param {Fixproofchartcaption2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartcaption2 = /** @type {((inputs?: Fixproofchartcaption2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartcaption2(inputs) + if (locale === "zh") return zh_fixproofchartcaption2(inputs) + if (locale === "ja") return ja_fixproofchartcaption2(inputs) + if (locale === "ko") return ko_fixproofchartcaption2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartcaption2(inputs) + if (locale === "de") return de_fixproofchartcaption2(inputs) + if (locale === "fr") return fr_fixproofchartcaption2(inputs) + if (locale === "uk") return uk_fixproofchartcaption2(inputs) + return en_fixproofchartcaption2(inputs) +}); +export { fixproofchartcaption2 as "fixproofChartCaption" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartheading2.js b/apps/web/src/paraglide/messages/fixproofchartheading2.js new file mode 100644 index 000000000..0e88ad5cd --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartheading2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofchartheading2Inputs */ + +const en_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Time against index`) +}; + +const es_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tiempo frente al índice`) +}; + +const zh_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`耗时与指数`) +}; + +const ja_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`所要時間と指数`) +}; + +const ko_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`시간과 지수`) +}; + +const zh_hant1_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`耗時與指數`) +}; + +const de_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Zeit gegen Index`) +}; + +const fr_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Temps et indice`) +}; + +const uk_fixproofchartheading2 = /** @type {(inputs: Fixproofchartheading2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Час і індекс`) +}; + +/** +* | output | +* | --- | +* | "Time against index" | +* +* @param {Fixproofchartheading2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartheading2 = /** @type {((inputs?: Fixproofchartheading2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartheading2(inputs) + if (locale === "zh") return zh_fixproofchartheading2(inputs) + if (locale === "ja") return ja_fixproofchartheading2(inputs) + if (locale === "ko") return ko_fixproofchartheading2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartheading2(inputs) + if (locale === "de") return de_fixproofchartheading2(inputs) + if (locale === "fr") return fr_fixproofchartheading2(inputs) + if (locale === "uk") return uk_fixproofchartheading2(inputs) + return en_fixproofchartheading2(inputs) +}); +export { fixproofchartheading2 as "fixproofChartHeading" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartlegendaria3.js b/apps/web/src/paraglide/messages/fixproofchartlegendaria3.js new file mode 100644 index 000000000..9a14316d8 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartlegendaria3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofchartlegendaria3Inputs */ + +const en_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Vendors`) +}; + +const es_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Proveedores`) +}; + +const zh_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`厂商`) +}; + +const ja_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`ベンダー`) +}; + +const ko_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`벤더`) +}; + +const zh_hant1_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`廠商`) +}; + +const de_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Anbieter`) +}; + +const fr_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fournisseurs`) +}; + +const uk_fixproofchartlegendaria3 = /** @type {(inputs: Fixproofchartlegendaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Постачальники`) +}; + +/** +* | output | +* | --- | +* | "Vendors" | +* +* @param {Fixproofchartlegendaria3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartlegendaria3 = /** @type {((inputs?: Fixproofchartlegendaria3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartlegendaria3(inputs) + if (locale === "zh") return zh_fixproofchartlegendaria3(inputs) + if (locale === "ja") return ja_fixproofchartlegendaria3(inputs) + if (locale === "ko") return ko_fixproofchartlegendaria3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartlegendaria3(inputs) + if (locale === "de") return de_fixproofchartlegendaria3(inputs) + if (locale === "fr") return fr_fixproofchartlegendaria3(inputs) + if (locale === "uk") return uk_fixproofchartlegendaria3(inputs) + return en_fixproofchartlegendaria3(inputs) +}); +export { fixproofchartlegendaria3 as "fixproofChartLegendAria" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartmetricaria3.js b/apps/web/src/paraglide/messages/fixproofchartmetricaria3.js new file mode 100644 index 000000000..994e102fb --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartmetricaria3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofchartmetricaria3Inputs */ + +const en_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Chart metric`) +}; + +const es_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Métrica del gráfico`) +}; + +const zh_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`图表指标`) +}; + +const ja_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`グラフの指標`) +}; + +const ko_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`차트 지표`) +}; + +const zh_hant1_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`圖表指標`) +}; + +const de_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Metrik des Diagramms`) +}; + +const fr_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Métrique du graphique`) +}; + +const uk_fixproofchartmetricaria3 = /** @type {(inputs: Fixproofchartmetricaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Метрика діаграми`) +}; + +/** +* | output | +* | --- | +* | "Chart metric" | +* +* @param {Fixproofchartmetricaria3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartmetricaria3 = /** @type {((inputs?: Fixproofchartmetricaria3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartmetricaria3(inputs) + if (locale === "zh") return zh_fixproofchartmetricaria3(inputs) + if (locale === "ja") return ja_fixproofchartmetricaria3(inputs) + if (locale === "ko") return ko_fixproofchartmetricaria3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartmetricaria3(inputs) + if (locale === "de") return de_fixproofchartmetricaria3(inputs) + if (locale === "fr") return fr_fixproofchartmetricaria3(inputs) + if (locale === "uk") return uk_fixproofchartmetricaria3(inputs) + return en_fixproofchartmetricaria3(inputs) +}); +export { fixproofchartmetricaria3 as "fixproofChartMetricAria" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartnote2.js b/apps/web/src/paraglide/messages/fixproofchartnote2.js new file mode 100644 index 000000000..d45b67917 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartnote2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofchartnote2Inputs */ + +const en_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`faster + higher ↗`) +}; + +const es_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`más rápido + más alto ↗`) +}; + +const zh_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`更快 + 更高 ↗`) +}; + +const ja_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`速い + 高い ↗`) +}; + +const ko_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`빠를수록 + 높을수록 ↗`) +}; + +const zh_hant1_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`更快 + 更高 ↗`) +}; + +const de_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`schneller + höher ↗`) +}; + +const fr_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`plus rapide + plus haut ↗`) +}; + +const uk_fixproofchartnote2 = /** @type {(inputs: Fixproofchartnote2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`швидше + вище ↗`) +}; + +/** +* | output | +* | --- | +* | "faster + higher ↗" | +* +* @param {Fixproofchartnote2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartnote2 = /** @type {((inputs?: Fixproofchartnote2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartnote2(inputs) + if (locale === "zh") return zh_fixproofchartnote2(inputs) + if (locale === "ja") return ja_fixproofchartnote2(inputs) + if (locale === "ko") return ko_fixproofchartnote2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartnote2(inputs) + if (locale === "de") return de_fixproofchartnote2(inputs) + if (locale === "fr") return fr_fixproofchartnote2(inputs) + if (locale === "uk") return uk_fixproofchartnote2(inputs) + return en_fixproofchartnote2(inputs) +}); +export { fixproofchartnote2 as "fixproofChartNote" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartpointaria3.js b/apps/web/src/paraglide/messages/fixproofchartpointaria3.js new file mode 100644 index 000000000..ebfe40aa1 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartpointaria3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{ model: NonNullable, metric: NonNullable, value: NonNullable, minutes: NonNullable }} Fixproofchartpointaria3Inputs */ + +const en_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model}, ${i?.metric} ${i?.value}, ${i?.minutes} median minutes`) +}; + +const es_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model}, ${i?.metric} ${i?.value}, ${i?.minutes} minutos de mediana`) +}; + +const zh_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model},${i?.metric} ${i?.value},中位数 ${i?.minutes} 分钟`) +}; + +const ja_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model}、${i?.metric} ${i?.value}、中央値 ${i?.minutes} 分`) +}; + +const ko_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model}, ${i?.metric} ${i?.value}, 중앙값 ${i?.minutes}분`) +}; + +const zh_hant1_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model},${i?.metric} ${i?.value},中位數 ${i?.minutes} 分鐘`) +}; + +const de_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model}, ${i?.metric} ${i?.value}, ${i?.minutes} Median-Minuten`) +}; + +const fr_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model}, ${i?.metric} ${i?.value}, ${i?.minutes} minutes médianes`) +}; + +const uk_fixproofchartpointaria3 = /** @type {(inputs: Fixproofchartpointaria3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.model}, ${i?.metric} ${i?.value}, медіана ${i?.minutes} хвилин`) +}; + +/** +* | output | +* | --- | +* | "{model}, {metric} {value}, {minutes} median minutes" | +* +* @param {Fixproofchartpointaria3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartpointaria3 = /** @type {((inputs: Fixproofchartpointaria3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartpointaria3(inputs) + if (locale === "zh") return zh_fixproofchartpointaria3(inputs) + if (locale === "ja") return ja_fixproofchartpointaria3(inputs) + if (locale === "ko") return ko_fixproofchartpointaria3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartpointaria3(inputs) + if (locale === "de") return de_fixproofchartpointaria3(inputs) + if (locale === "fr") return fr_fixproofchartpointaria3(inputs) + if (locale === "uk") return uk_fixproofchartpointaria3(inputs) + return en_fixproofchartpointaria3(inputs) +}); +export { fixproofchartpointaria3 as "fixproofChartPointAria" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofchartregionaria3.js b/apps/web/src/paraglide/messages/fixproofchartregionaria3.js new file mode 100644 index 000000000..009ba10a3 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofchartregionaria3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofchartregionaria3Inputs */ + +const en_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof scatter chart`) +}; + +const es_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Gráfico de dispersión de Fixproof`) +}; + +const zh_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof 散点图`) +}; + +const ja_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof 散布図`) +}; + +const ko_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof 산점도`) +}; + +const zh_hant1_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof 散佈圖`) +}; + +const de_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof-Streudiagramm`) +}; + +const fr_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Nuage de points Fixproof`) +}; + +const uk_fixproofchartregionaria3 = /** @type {(inputs: Fixproofchartregionaria3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Точкова діаграма Fixproof`) +}; + +/** +* | output | +* | --- | +* | "Fixproof scatter chart" | +* +* @param {Fixproofchartregionaria3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofchartregionaria3 = /** @type {((inputs?: Fixproofchartregionaria3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofchartregionaria3(inputs) + if (locale === "zh") return zh_fixproofchartregionaria3(inputs) + if (locale === "ja") return ja_fixproofchartregionaria3(inputs) + if (locale === "ko") return ko_fixproofchartregionaria3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofchartregionaria3(inputs) + if (locale === "de") return de_fixproofchartregionaria3(inputs) + if (locale === "fr") return fr_fixproofchartregionaria3(inputs) + if (locale === "uk") return uk_fixproofchartregionaria3(inputs) + return en_fixproofchartregionaria3(inputs) +}); +export { fixproofchartregionaria3 as "fixproofChartRegionAria" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofclaim1.js b/apps/web/src/paraglide/messages/fixproofclaim1.js new file mode 100644 index 000000000..916e0102c --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofclaim1.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofclaim1Inputs */ + +const en_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Real issues from private and public codebases, sealed. Hidden tests decide.`) +}; + +const es_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Errores reales de bases de código privadas y públicas, sellados. Deciden las pruebas ocultas.`) +}; + +const zh_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`来自私有和公开代码库的真实问题,全部封闭。由隐藏测试判定。`) +}; + +const ja_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`非公開および公開コードベースの実際の不具合を封印。判定するのは非公開テストです。`) +}; + +const ko_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`비공개 및 공개 코드베이스의 실제 이슈를 봉인했습니다. 판정은 비공개 테스트가 합니다.`) +}; + +const zh_hant1_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`來自私有和公開程式碼庫的真實問題,全部封閉。由隱藏測試判定。`) +}; + +const de_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Echte Fehler aus privaten und öffentlichen Codebasen, versiegelt. Verborgene Tests entscheiden.`) +}; + +const fr_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Des bugs réels issus de bases de code privées et publiques, scellés. Ce sont les tests cachés qui tranchent.`) +}; + +const uk_fixproofclaim1 = /** @type {(inputs: Fixproofclaim1Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Реальні помилки з приватних і публічних кодових баз, закриті. Вирішують приховані тести.`) +}; + +/** +* | output | +* | --- | +* | "Real issues from private and public codebases, sealed. Hidden tests decide." | +* +* @param {Fixproofclaim1Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofclaim1 = /** @type {((inputs?: Fixproofclaim1Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofclaim1(inputs) + if (locale === "zh") return zh_fixproofclaim1(inputs) + if (locale === "ja") return ja_fixproofclaim1(inputs) + if (locale === "ko") return ko_fixproofclaim1(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofclaim1(inputs) + if (locale === "de") return de_fixproofclaim1(inputs) + if (locale === "fr") return fr_fixproofclaim1(inputs) + if (locale === "uk") return uk_fixproofclaim1(inputs) + return en_fixproofclaim1(inputs) +}); +export { fixproofclaim1 as "fixproofClaim" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolclaimedonly3.js b/apps/web/src/paraglide/messages/fixproofcolclaimedonly3.js new file mode 100644 index 000000000..7a9e48d92 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolclaimedonly3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolclaimedonly3Inputs */ + +const en_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Claimed, not done`) +}; + +const es_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Declarado, no hecho`) +}; + +const zh_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`只是声称,并未完成`) +}; + +const ja_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`主張のみ、未実施`) +}; + +const ko_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`주장만 하고 안 함`) +}; + +const zh_hant1_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`只是聲稱,並未完成`) +}; + +const de_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Behauptet, nicht erledigt`) +}; + +const fr_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Annoncé, pas fait`) +}; + +const uk_fixproofcolclaimedonly3 = /** @type {(inputs: Fixproofcolclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Заявлено, не зроблено`) +}; + +/** +* | output | +* | --- | +* | "Claimed, not done" | +* +* @param {Fixproofcolclaimedonly3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolclaimedonly3 = /** @type {((inputs?: Fixproofcolclaimedonly3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolclaimedonly3(inputs) + if (locale === "zh") return zh_fixproofcolclaimedonly3(inputs) + if (locale === "ja") return ja_fixproofcolclaimedonly3(inputs) + if (locale === "ko") return ko_fixproofcolclaimedonly3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolclaimedonly3(inputs) + if (locale === "de") return de_fixproofcolclaimedonly3(inputs) + if (locale === "fr") return fr_fixproofcolclaimedonly3(inputs) + if (locale === "uk") return uk_fixproofcolclaimedonly3(inputs) + return en_fixproofcolclaimedonly3(inputs) +}); +export { fixproofcolclaimedonly3 as "fixproofColClaimedOnly" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcoleffort2.js b/apps/web/src/paraglide/messages/fixproofcoleffort2.js new file mode 100644 index 000000000..f13a5a354 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcoleffort2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcoleffort2Inputs */ + +const en_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Effort`) +}; + +const es_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Esfuerzo`) +}; + +const zh_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`推理强度`) +}; + +const ja_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`推論強度`) +}; + +const ko_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`추론 강도`) +}; + +const zh_hant1_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`推理強度`) +}; + +const de_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Effort`) +}; + +const fr_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Effort`) +}; + +const uk_fixproofcoleffort2 = /** @type {(inputs: Fixproofcoleffort2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Рівень зусиль`) +}; + +/** +* | output | +* | --- | +* | "Effort" | +* +* @param {Fixproofcoleffort2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcoleffort2 = /** @type {((inputs?: Fixproofcoleffort2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcoleffort2(inputs) + if (locale === "zh") return zh_fixproofcoleffort2(inputs) + if (locale === "ja") return ja_fixproofcoleffort2(inputs) + if (locale === "ko") return ko_fixproofcoleffort2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcoleffort2(inputs) + if (locale === "de") return de_fixproofcoleffort2(inputs) + if (locale === "fr") return fr_fixproofcoleffort2(inputs) + if (locale === "uk") return uk_fixproofcoleffort2(inputs) + return en_fixproofcoleffort2(inputs) +}); +export { fixproofcoleffort2 as "fixproofColEffort" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolharness2.js b/apps/web/src/paraglide/messages/fixproofcolharness2.js new file mode 100644 index 000000000..fa2d64da7 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolharness2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolharness2Inputs */ + +const en_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Harness`) +}; + +const es_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Harness`) +}; + +const zh_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Harness`) +}; + +const ja_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`ハーネス`) +}; + +const ko_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`하네스`) +}; + +const zh_hant1_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Harness`) +}; + +const de_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Harness`) +}; + +const fr_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Harness`) +}; + +const uk_fixproofcolharness2 = /** @type {(inputs: Fixproofcolharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Harness`) +}; + +/** +* | output | +* | --- | +* | "Harness" | +* +* @param {Fixproofcolharness2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolharness2 = /** @type {((inputs?: Fixproofcolharness2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolharness2(inputs) + if (locale === "zh") return zh_fixproofcolharness2(inputs) + if (locale === "ja") return ja_fixproofcolharness2(inputs) + if (locale === "ko") return ko_fixproofcolharness2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolharness2(inputs) + if (locale === "de") return de_fixproofcolharness2(inputs) + if (locale === "fr") return fr_fixproofcolharness2(inputs) + if (locale === "uk") return uk_fixproofcolharness2(inputs) + return en_fixproofcolharness2(inputs) +}); +export { fixproofcolharness2 as "fixproofColHarness" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolmedianminutes3.js b/apps/web/src/paraglide/messages/fixproofcolmedianminutes3.js new file mode 100644 index 000000000..0a3695937 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolmedianminutes3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolmedianminutes3Inputs */ + +const en_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Median minutes`) +}; + +const es_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Mediana de minutos`) +}; + +const zh_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`耗时中位数(分钟)`) +}; + +const ja_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`所要時間の中央値 (分)`) +}; + +const ko_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`중앙값 분`) +}; + +const zh_hant1_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`耗時中位數(分鐘)`) +}; + +const de_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Median-Minuten`) +}; + +const fr_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Minutes médianes`) +}; + +const uk_fixproofcolmedianminutes3 = /** @type {(inputs: Fixproofcolmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Медіана хвилин`) +}; + +/** +* | output | +* | --- | +* | "Median minutes" | +* +* @param {Fixproofcolmedianminutes3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolmedianminutes3 = /** @type {((inputs?: Fixproofcolmedianminutes3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolmedianminutes3(inputs) + if (locale === "zh") return zh_fixproofcolmedianminutes3(inputs) + if (locale === "ja") return ja_fixproofcolmedianminutes3(inputs) + if (locale === "ko") return ko_fixproofcolmedianminutes3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolmedianminutes3(inputs) + if (locale === "de") return de_fixproofcolmedianminutes3(inputs) + if (locale === "fr") return fr_fixproofcolmedianminutes3(inputs) + if (locale === "uk") return uk_fixproofcolmedianminutes3(inputs) + return en_fixproofcolmedianminutes3(inputs) +}); +export { fixproofcolmedianminutes3 as "fixproofColMedianMinutes" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolmodel2.js b/apps/web/src/paraglide/messages/fixproofcolmodel2.js new file mode 100644 index 000000000..49d0532ce --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolmodel2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolmodel2Inputs */ + +const en_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Model`) +}; + +const es_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Modelo`) +}; + +const zh_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`模型`) +}; + +const ja_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`モデル`) +}; + +const ko_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`모델`) +}; + +const zh_hant1_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`模型`) +}; + +const de_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Modell`) +}; + +const fr_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Modèle`) +}; + +const uk_fixproofcolmodel2 = /** @type {(inputs: Fixproofcolmodel2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Модель`) +}; + +/** +* | output | +* | --- | +* | "Model" | +* +* @param {Fixproofcolmodel2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolmodel2 = /** @type {((inputs?: Fixproofcolmodel2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolmodel2(inputs) + if (locale === "zh") return zh_fixproofcolmodel2(inputs) + if (locale === "ja") return ja_fixproofcolmodel2(inputs) + if (locale === "ko") return ko_fixproofcolmodel2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolmodel2(inputs) + if (locale === "de") return de_fixproofcolmodel2(inputs) + if (locale === "fr") return fr_fixproofcolmodel2(inputs) + if (locale === "uk") return uk_fixproofcolmodel2(inputs) + return en_fixproofcolmodel2(inputs) +}); +export { fixproofcolmodel2 as "fixproofColModel" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolprogressindex3.js b/apps/web/src/paraglide/messages/fixproofcolprogressindex3.js new file mode 100644 index 000000000..bcdc6b6e0 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolprogressindex3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolprogressindex3Inputs */ + +const en_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Progress index`) +}; + +const es_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Índice Progress`) +}; + +const zh_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Progress 指数`) +}; + +const ja_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Progress 指数`) +}; + +const ko_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Progress 지수`) +}; + +const zh_hant1_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Progress 指數`) +}; + +const de_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Progress-Index`) +}; + +const fr_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Indice Progress`) +}; + +const uk_fixproofcolprogressindex3 = /** @type {(inputs: Fixproofcolprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Індекс Progress`) +}; + +/** +* | output | +* | --- | +* | "Progress index" | +* +* @param {Fixproofcolprogressindex3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolprogressindex3 = /** @type {((inputs?: Fixproofcolprogressindex3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolprogressindex3(inputs) + if (locale === "zh") return zh_fixproofcolprogressindex3(inputs) + if (locale === "ja") return ja_fixproofcolprogressindex3(inputs) + if (locale === "ko") return ko_fixproofcolprogressindex3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolprogressindex3(inputs) + if (locale === "de") return de_fixproofcolprogressindex3(inputs) + if (locale === "fr") return fr_fixproofcolprogressindex3(inputs) + if (locale === "uk") return uk_fixproofcolprogressindex3(inputs) + return en_fixproofcolprogressindex3(inputs) +}); +export { fixproofcolprogressindex3 as "fixproofColProgressIndex" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolregressions2.js b/apps/web/src/paraglide/messages/fixproofcolregressions2.js new file mode 100644 index 000000000..a812d4d23 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolregressions2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolregressions2Inputs */ + +const en_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Regressions`) +}; + +const es_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Regresiones`) +}; + +const zh_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`回归`) +}; + +const ja_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`リグレッション`) +}; + +const ko_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`회귀`) +}; + +const zh_hant1_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`迴歸`) +}; + +const de_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Regressionen`) +}; + +const fr_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Régressions`) +}; + +const uk_fixproofcolregressions2 = /** @type {(inputs: Fixproofcolregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Регресії`) +}; + +/** +* | output | +* | --- | +* | "Regressions" | +* +* @param {Fixproofcolregressions2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolregressions2 = /** @type {((inputs?: Fixproofcolregressions2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolregressions2(inputs) + if (locale === "zh") return zh_fixproofcolregressions2(inputs) + if (locale === "ja") return ja_fixproofcolregressions2(inputs) + if (locale === "ko") return ko_fixproofcolregressions2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolregressions2(inputs) + if (locale === "de") return de_fixproofcolregressions2(inputs) + if (locale === "fr") return fr_fixproofcolregressions2(inputs) + if (locale === "uk") return uk_fixproofcolregressions2(inputs) + return en_fixproofcolregressions2(inputs) +}); +export { fixproofcolregressions2 as "fixproofColRegressions" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolresolvedindex3.js b/apps/web/src/paraglide/messages/fixproofcolresolvedindex3.js new file mode 100644 index 000000000..cf4a8f5c2 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolresolvedindex3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolresolvedindex3Inputs */ + +const en_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Resolved index`) +}; + +const es_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Índice Resolved`) +}; + +const zh_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Resolved 指数`) +}; + +const ja_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Resolved 指数`) +}; + +const ko_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Resolved 지수`) +}; + +const zh_hant1_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Resolved 指數`) +}; + +const de_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Resolved-Index`) +}; + +const fr_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Indice Resolved`) +}; + +const uk_fixproofcolresolvedindex3 = /** @type {(inputs: Fixproofcolresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Індекс Resolved`) +}; + +/** +* | output | +* | --- | +* | "Resolved index" | +* +* @param {Fixproofcolresolvedindex3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolresolvedindex3 = /** @type {((inputs?: Fixproofcolresolvedindex3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolresolvedindex3(inputs) + if (locale === "zh") return zh_fixproofcolresolvedindex3(inputs) + if (locale === "ja") return ja_fixproofcolresolvedindex3(inputs) + if (locale === "ko") return ko_fixproofcolresolvedindex3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolresolvedindex3(inputs) + if (locale === "de") return de_fixproofcolresolvedindex3(inputs) + if (locale === "fr") return fr_fixproofcolresolvedindex3(inputs) + if (locale === "uk") return uk_fixproofcolresolvedindex3(inputs) + return en_fixproofcolresolvedindex3(inputs) +}); +export { fixproofcolresolvedindex3 as "fixproofColResolvedIndex" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolrundate3.js b/apps/web/src/paraglide/messages/fixproofcolrundate3.js new file mode 100644 index 000000000..a53a0808c --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolrundate3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolrundate3Inputs */ + +const en_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Run date`) +}; + +const es_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fecha de ejecución`) +}; + +const zh_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`运行日期`) +}; + +const ja_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`実行日`) +}; + +const ko_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`실행 날짜`) +}; + +const zh_hant1_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`執行日期`) +}; + +const de_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Laufdatum`) +}; + +const fr_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Date d'exécution`) +}; + +const uk_fixproofcolrundate3 = /** @type {(inputs: Fixproofcolrundate3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Дата запуску`) +}; + +/** +* | output | +* | --- | +* | "Run date" | +* +* @param {Fixproofcolrundate3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolrundate3 = /** @type {((inputs?: Fixproofcolrundate3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolrundate3(inputs) + if (locale === "zh") return zh_fixproofcolrundate3(inputs) + if (locale === "ja") return ja_fixproofcolrundate3(inputs) + if (locale === "ko") return ko_fixproofcolrundate3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolrundate3(inputs) + if (locale === "de") return de_fixproofcolrundate3(inputs) + if (locale === "fr") return fr_fixproofcolrundate3(inputs) + if (locale === "uk") return uk_fixproofcolrundate3(inputs) + return en_fixproofcolrundate3(inputs) +}); +export { fixproofcolrundate3 as "fixproofColRunDate" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcolsolvedovergraded4.js b/apps/web/src/paraglide/messages/fixproofcolsolvedovergraded4.js new file mode 100644 index 000000000..cf6e545e0 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcolsolvedovergraded4.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcolsolvedovergraded4Inputs */ + +const en_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Solved / graded`) +}; + +const es_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Resueltas / evaluadas`) +}; + +const zh_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`已解决 / 已评测`) +}; + +const ja_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`解決 / 採点`) +}; + +const ko_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`해결 / 채점`) +}; + +const zh_hant1_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`已解決 / 已評測`) +}; + +const de_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Gelöst / bewertet`) +}; + +const fr_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Résolues / évaluées`) +}; + +const uk_fixproofcolsolvedovergraded4 = /** @type {(inputs: Fixproofcolsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Вирішено / оцінено`) +}; + +/** +* | output | +* | --- | +* | "Solved / graded" | +* +* @param {Fixproofcolsolvedovergraded4Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcolsolvedovergraded4 = /** @type {((inputs?: Fixproofcolsolvedovergraded4Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcolsolvedovergraded4(inputs) + if (locale === "zh") return zh_fixproofcolsolvedovergraded4(inputs) + if (locale === "ja") return ja_fixproofcolsolvedovergraded4(inputs) + if (locale === "ko") return ko_fixproofcolsolvedovergraded4(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcolsolvedovergraded4(inputs) + if (locale === "de") return de_fixproofcolsolvedovergraded4(inputs) + if (locale === "fr") return fr_fixproofcolsolvedovergraded4(inputs) + if (locale === "uk") return uk_fixproofcolsolvedovergraded4(inputs) + return en_fixproofcolsolvedovergraded4(inputs) +}); +export { fixproofcolsolvedovergraded4 as "fixproofColSolvedOverGraded" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcoltestedits3.js b/apps/web/src/paraglide/messages/fixproofcoltestedits3.js new file mode 100644 index 000000000..35f492d08 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcoltestedits3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcoltestedits3Inputs */ + +const en_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Test edits reverted`) +}; + +const es_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Cambios en pruebas revertidos`) +}; + +const zh_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`已还原的测试改动`) +}; + +const ja_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`差し戻したテスト変更`) +}; + +const ko_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`되돌린 테스트 수정`) +}; + +const zh_hant1_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`已還原的測試改動`) +}; + +const de_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Test-Änderungen zurückgesetzt`) +}; + +const fr_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Modifications de tests annulées`) +}; + +const uk_fixproofcoltestedits3 = /** @type {(inputs: Fixproofcoltestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Скасовані зміни в тестах`) +}; + +/** +* | output | +* | --- | +* | "Test edits reverted" | +* +* @param {Fixproofcoltestedits3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcoltestedits3 = /** @type {((inputs?: Fixproofcoltestedits3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcoltestedits3(inputs) + if (locale === "zh") return zh_fixproofcoltestedits3(inputs) + if (locale === "ja") return ja_fixproofcoltestedits3(inputs) + if (locale === "ko") return ko_fixproofcoltestedits3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcoltestedits3(inputs) + if (locale === "de") return de_fixproofcoltestedits3(inputs) + if (locale === "fr") return fr_fixproofcoltestedits3(inputs) + if (locale === "uk") return uk_fixproofcoltestedits3(inputs) + return en_fixproofcoltestedits3(inputs) +}); +export { fixproofcoltestedits3 as "fixproofColTestEdits" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofcoltrials2.js b/apps/web/src/paraglide/messages/fixproofcoltrials2.js new file mode 100644 index 000000000..a05ecaba9 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofcoltrials2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofcoltrials2Inputs */ + +const en_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Trials`) +}; + +const es_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Intentos`) +}; + +const zh_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`试验次数`) +}; + +const ja_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`試行数`) +}; + +const ko_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`시도`) +}; + +const zh_hant1_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`試驗次數`) +}; + +const de_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Versuche`) +}; + +const fr_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Essais`) +}; + +const uk_fixproofcoltrials2 = /** @type {(inputs: Fixproofcoltrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Спроби`) +}; + +/** +* | output | +* | --- | +* | "Trials" | +* +* @param {Fixproofcoltrials2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofcoltrials2 = /** @type {((inputs?: Fixproofcoltrials2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofcoltrials2(inputs) + if (locale === "zh") return zh_fixproofcoltrials2(inputs) + if (locale === "ja") return ja_fixproofcoltrials2(inputs) + if (locale === "ko") return ko_fixproofcoltrials2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofcoltrials2(inputs) + if (locale === "de") return de_fixproofcoltrials2(inputs) + if (locale === "fr") return fr_fixproofcoltrials2(inputs) + if (locale === "uk") return uk_fixproofcoltrials2(inputs) + return en_fixproofcoltrials2(inputs) +}); +export { fixproofcoltrials2 as "fixproofColTrials" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefclaimedonly3.js b/apps/web/src/paraglide/messages/fixproofdefclaimedonly3.js new file mode 100644 index 000000000..e5804ad1a --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefclaimedonly3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdefclaimedonly3Inputs */ + +const en_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Runs where the agent's summary claimed edits that never reached disk.`) +}; + +const es_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Ejecuciones en las que el resumen del agente declaró cambios que nunca llegaron al disco.`) +}; + +const zh_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`代理在总结里声称做了改动,但这些改动从未落到磁盘上的运行。`) +}; + +const ja_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`エージェントの要約が、ディスクに届かなかった変更を主張した実行です。`) +}; + +const ko_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`에이전트 요약이 디스크에 반영되지 않은 수정을 했다고 주장한 실행입니다.`) +}; + +const zh_hant1_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`代理程式在總結裡聲稱做了改動,但這些改動從未寫入磁碟的執行。`) +}; + +const de_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Läufe, in denen die Zusammenfassung des Agenten Änderungen behauptet hat, die nie auf der Festplatte gelandet sind.`) +}; + +const fr_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Exécutions où le résumé de l'agent annonçait des modifications qui n'ont jamais atteint le disque.`) +}; + +const uk_fixproofdefclaimedonly3 = /** @type {(inputs: Fixproofdefclaimedonly3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Запуски, де підсумок агента заявляв про зміни, які так і не потрапили на диск.`) +}; + +/** +* | output | +* | --- | +* | "Runs where the agent's summary claimed edits that never reached disk." | +* +* @param {Fixproofdefclaimedonly3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefclaimedonly3 = /** @type {((inputs?: Fixproofdefclaimedonly3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefclaimedonly3(inputs) + if (locale === "zh") return zh_fixproofdefclaimedonly3(inputs) + if (locale === "ja") return ja_fixproofdefclaimedonly3(inputs) + if (locale === "ko") return ko_fixproofdefclaimedonly3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefclaimedonly3(inputs) + if (locale === "de") return de_fixproofdefclaimedonly3(inputs) + if (locale === "fr") return fr_fixproofdefclaimedonly3(inputs) + if (locale === "uk") return uk_fixproofdefclaimedonly3(inputs) + return en_fixproofdefclaimedonly3(inputs) +}); +export { fixproofdefclaimedonly3 as "fixproofDefClaimedOnly" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefharness2.js b/apps/web/src/paraglide/messages/fixproofdefharness2.js new file mode 100644 index 000000000..908d690ab --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefharness2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdefharness2Inputs */ + +const en_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`The agent CLI that drove the model.`) +}; + +const es_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`La CLI de agente que condujo el modelo.`) +}; + +const zh_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`驱动模型的代理 CLI。`) +}; + +const ja_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`モデルを動かしたエージェント CLI です。`) +}; + +const ko_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`모델을 구동한 에이전트 CLI입니다.`) +}; + +const zh_hant1_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`驅動模型的代理程式 CLI。`) +}; + +const de_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Das Agenten-CLI, das das Modell gesteuert hat.`) +}; + +const fr_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`La CLI d'agent qui a piloté le modèle.`) +}; + +const uk_fixproofdefharness2 = /** @type {(inputs: Fixproofdefharness2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`CLI агента, який керував моделлю.`) +}; + +/** +* | output | +* | --- | +* | "The agent CLI that drove the model." | +* +* @param {Fixproofdefharness2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefharness2 = /** @type {((inputs?: Fixproofdefharness2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefharness2(inputs) + if (locale === "zh") return zh_fixproofdefharness2(inputs) + if (locale === "ja") return ja_fixproofdefharness2(inputs) + if (locale === "ko") return ko_fixproofdefharness2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefharness2(inputs) + if (locale === "de") return de_fixproofdefharness2(inputs) + if (locale === "fr") return fr_fixproofdefharness2(inputs) + if (locale === "uk") return uk_fixproofdefharness2(inputs) + return en_fixproofdefharness2(inputs) +}); +export { fixproofdefharness2 as "fixproofDefHarness" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefinitionaria2.js b/apps/web/src/paraglide/messages/fixproofdefinitionaria2.js new file mode 100644 index 000000000..0476194f2 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefinitionaria2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{ column: NonNullable }} Fixproofdefinitionaria2Inputs */ + +const en_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`What does ${i?.column} mean?`) +}; + +const es_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`¿Qué significa ${i?.column}?`) +}; + +const zh_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.column} 是什么意思?`) +}; + +const ja_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.column} の意味は?`) +}; + +const ko_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.column}의 의미는 무엇인가요?`) +}; + +const zh_hant1_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.column} 是什麼意思?`) +}; + +const de_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Was bedeutet ${i?.column}?`) +}; + +const fr_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Que signifie ${i?.column} ?`) +}; + +const uk_fixproofdefinitionaria2 = /** @type {(inputs: Fixproofdefinitionaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Що означає ${i?.column}?`) +}; + +/** +* | output | +* | --- | +* | "What does {column} mean?" | +* +* @param {Fixproofdefinitionaria2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefinitionaria2 = /** @type {((inputs: Fixproofdefinitionaria2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefinitionaria2(inputs) + if (locale === "zh") return zh_fixproofdefinitionaria2(inputs) + if (locale === "ja") return ja_fixproofdefinitionaria2(inputs) + if (locale === "ko") return ko_fixproofdefinitionaria2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefinitionaria2(inputs) + if (locale === "de") return de_fixproofdefinitionaria2(inputs) + if (locale === "fr") return fr_fixproofdefinitionaria2(inputs) + if (locale === "uk") return uk_fixproofdefinitionaria2(inputs) + return en_fixproofdefinitionaria2(inputs) +}); +export { fixproofdefinitionaria2 as "fixproofDefinitionAria" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefmedianminutes3.js b/apps/web/src/paraglide/messages/fixproofdefmedianminutes3.js new file mode 100644 index 000000000..1954c85e4 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefmedianminutes3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdefmedianminutes3Inputs */ + +const en_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Median wall-clock minutes the agent worked before it stopped or hit the 30 minute cap.`) +}; + +const es_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Mediana de minutos reales que el agente trabajó antes de parar o de llegar al límite de 30 minutos.`) +}; + +const zh_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`代理在停止或触及 30 分钟上限之前实际工作时长的中位数(分钟)。`) +}; + +const ja_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`エージェントが停止するか 30 分の上限に達するまでに作業した実時間の中央値 (分) です。`) +}; + +const ko_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`에이전트가 멈추거나 30분 제한에 도달할 때까지 작업한 실제 시간의 중앙값입니다.`) +}; + +const zh_hant1_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`代理程式在停止或觸及 30 分鐘上限之前實際工作時長的中位數(分鐘)。`) +}; + +const de_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Median der tatsächlich verstrichenen Minuten, die der Agent gearbeitet hat, bevor er gestoppt hat oder das Limit von 30 Minuten erreicht war.`) +}; + +const fr_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Médiane des minutes réelles travaillées par l'agent avant qu'il s'arrête ou atteigne la limite de 30 minutes.`) +}; + +const uk_fixproofdefmedianminutes3 = /** @type {(inputs: Fixproofdefmedianminutes3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Медіана реального часу в хвилинах, який агент працював, доки не зупинився або не досяг ліміту в 30 хвилин.`) +}; + +/** +* | output | +* | --- | +* | "Median wall-clock minutes the agent worked before it stopped or hit the 30 minute cap." | +* +* @param {Fixproofdefmedianminutes3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefmedianminutes3 = /** @type {((inputs?: Fixproofdefmedianminutes3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefmedianminutes3(inputs) + if (locale === "zh") return zh_fixproofdefmedianminutes3(inputs) + if (locale === "ja") return ja_fixproofdefmedianminutes3(inputs) + if (locale === "ko") return ko_fixproofdefmedianminutes3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefmedianminutes3(inputs) + if (locale === "de") return de_fixproofdefmedianminutes3(inputs) + if (locale === "fr") return fr_fixproofdefmedianminutes3(inputs) + if (locale === "uk") return uk_fixproofdefmedianminutes3(inputs) + return en_fixproofdefmedianminutes3(inputs) +}); +export { fixproofdefmedianminutes3 as "fixproofDefMedianMinutes" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefprogressindex3.js b/apps/web/src/paraglide/messages/fixproofdefprogressindex3.js new file mode 100644 index 000000000..9b22d4368 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefprogressindex3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdefprogressindex3Inputs */ + +const en_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Weighted share of each task's requirements that were failing at the base commit and pass after the agent's patch, then difficulty-weighted across tasks. Each requirement carries a weight of 2 (core), 1 or 0.5 (peripheral); requirements already green at base and untested requirements are excluded from both the numerator and denominator.`) +}; + +const es_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Proporción ponderada de los requisitos de cada tarea que fallaban en el commit base y pasan tras el parche del agente, ponderada después por dificultad entre tareas. Cada requisito lleva un peso de 2 (central), 1 o 0,5 (periférico); los requisitos que ya pasaban en el commit base y los requisitos no probados se excluyen tanto del numerador como del denominador.`) +}; + +const zh_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每个任务中,在基线提交上失败、在代理的补丁之后通过的需求所占的加权比例,再按难度在任务之间加权。每条需求的权重为 2(核心)、1 或 0.5(外围);在基线提交上已经通过的需求和未经测试的需求均从分子和分母中排除。`) +}; + +const ja_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`各タスクの要件のうち、ベースコミットでは失敗しエージェントのパッチ後に成功するものの重み付き割合を求め、さらにタスク間で難易度による重み付けを行った値です。各要件の重みは 2 (中核)、1 または 0.5 (周辺) です。ベースコミットの時点ですでに成功していた要件と未テストの要件は、分子と分母の両方から除外します。`) +}; + +const ko_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`각 태스크의 요구사항 중 베이스 커밋에서 실패하고 에이전트의 패치 이후 통과한 비율을 가중해 구한 뒤, 태스크 사이에서 난이도로 다시 가중한 값입니다. 요구사항의 가중치는 2(핵심), 1 또는 0.5(주변)입니다. 베이스 커밋에서 이미 통과하던 요구사항과 테스트하지 않은 요구사항은 분자와 분모에서 모두 제외합니다.`) +}; + +const zh_hant1_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每個任務中,在基線提交上失敗、在代理程式的修補之後通過的需求所占的加權比例,再按難度在任務之間加權。每條需求的權重為 2(核心)、1 或 0.5(外圍);在基線提交上已經通過的需求和未經測試的需求均從分子和分母中排除。`) +}; + +const de_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Gewichteter Anteil der Anforderungen einer Aufgabe, die auf dem Basis-Commit fehlgeschlagen sind und nach dem Patch des Agenten bestehen, anschließend über alle Aufgaben nach Schwierigkeit gewichtet. Jede Anforderung hat ein Gewicht von 2 (Kern), 1 oder 0,5 (Rand); Anforderungen, die auf dem Basis-Commit schon grün waren, sowie ungetestete Anforderungen werden sowohl aus dem Zähler als auch aus dem Nenner ausgeschlossen.`) +}; + +const fr_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Part pondérée des exigences de chaque tâche qui échouaient au commit de base et passent après le correctif de l'agent, puis pondérée par la difficulté sur l'ensemble des tâches. Chaque exigence porte un poids de 2 (cœur), 1 ou 0,5 (périphérique) ; les exigences déjà satisfaites au commit de base et les exigences non testées sont exclues du numérateur comme du dénominateur.`) +}; + +const uk_fixproofdefprogressindex3 = /** @type {(inputs: Fixproofdefprogressindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Зважена частка вимог кожної задачі, які падали на базовому коміті й проходять після патча агента, далі зважена за складністю по всіх задачах. Кожна вимога має вагу 2 (основна), 1 або 0,5 (периферійна); вимоги, що вже проходили на базовому коміті, та неперевірені вимоги виключаються і з чисельника, і зі знаменника.`) +}; + +/** +* | output | +* | --- | +* | "Weighted share of each task's requirements that were failing at the base commit and pass after the agent's patch, then difficulty-weighted across tasks. Each..." | +* +* @param {Fixproofdefprogressindex3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefprogressindex3 = /** @type {((inputs?: Fixproofdefprogressindex3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefprogressindex3(inputs) + if (locale === "zh") return zh_fixproofdefprogressindex3(inputs) + if (locale === "ja") return ja_fixproofdefprogressindex3(inputs) + if (locale === "ko") return ko_fixproofdefprogressindex3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefprogressindex3(inputs) + if (locale === "de") return de_fixproofdefprogressindex3(inputs) + if (locale === "fr") return fr_fixproofdefprogressindex3(inputs) + if (locale === "uk") return uk_fixproofdefprogressindex3(inputs) + return en_fixproofdefprogressindex3(inputs) +}); +export { fixproofdefprogressindex3 as "fixproofDefProgressIndex" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefregressions2.js b/apps/web/src/paraglide/messages/fixproofdefregressions2.js new file mode 100644 index 000000000..465776eac --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefregressions2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdefregressions2Inputs */ + +const en_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Graded runs where the package's existing test suite stopped passing. A dash means at least one graded run has no regression result.`) +}; + +const es_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Ejecuciones evaluadas en las que la suite de pruebas existente del paquete dejó de pasar. Un guion indica que al menos una ejecución evaluada no tiene un resultado de regresiones.`) +}; + +const zh_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`评测运行中,包自带的测试套件不再通过的那些。 短横线表示至少一次已评测运行的回归结果未知。`) +}; + +const ja_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`パッケージ既存のテストスイートが通らなくなった採点済みの実行です。 ダッシュは、採点済みの実行のうち少なくとも1件でリグレッションの結果が不明であることを示します。`) +}; + +const ko_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`패키지의 기존 테스트 스위트가 더 이상 통과하지 않게 된 채점 실행입니다. 대시는 채점된 실행 중 하나 이상에서 회귀 결과를 알 수 없음을 뜻합니다.`) +}; + +const zh_hant1_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`評測執行中,套件自帶的測試套件不再通過的那些。 短橫線表示至少一次已評測執行的迴歸結果未知。`) +}; + +const de_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Bewertete Läufe, in denen die vorhandene Test-Suite des Pakets nicht mehr bestanden wurde. Ein Strich bedeutet, dass für mindestens einen bewerteten Lauf kein Regressionsergebnis vorliegt.`) +}; + +const fr_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Exécutions évaluées où la suite de tests existante du paquet a cessé de passer. Un tiret indique qu’au moins une exécution évaluée n’a pas de résultat de régression.`) +}; + +const uk_fixproofdefregressions2 = /** @type {(inputs: Fixproofdefregressions2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Оцінені запуски, де наявний набір тестів пакета перестав проходити. Риска означає, що принаймні для одного оціненого запуску немає результату перевірки регресій.`) +}; + +/** +* | output | +* | --- | +* | "Graded runs where the package's existing test suite stopped passing. A dash means at least one graded run has no regression result." | +* +* @param {Fixproofdefregressions2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefregressions2 = /** @type {((inputs?: Fixproofdefregressions2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefregressions2(inputs) + if (locale === "zh") return zh_fixproofdefregressions2(inputs) + if (locale === "ja") return ja_fixproofdefregressions2(inputs) + if (locale === "ko") return ko_fixproofdefregressions2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefregressions2(inputs) + if (locale === "de") return de_fixproofdefregressions2(inputs) + if (locale === "fr") return fr_fixproofdefregressions2(inputs) + if (locale === "uk") return uk_fixproofdefregressions2(inputs) + return en_fixproofdefregressions2(inputs) +}); +export { fixproofdefregressions2 as "fixproofDefRegressions" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefresolvedindex3.js b/apps/web/src/paraglide/messages/fixproofdefresolvedindex3.js new file mode 100644 index 000000000..0bd68cc16 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefresolvedindex3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdefresolvedindex3Inputs */ + +const en_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Difficulty-weighted share of tasks where every hidden check passed and no regression appeared. This is the headline number.`) +}; + +const es_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Proporción ponderada por dificultad de las tareas en las que pasaron todas las comprobaciones ocultas y no apareció ninguna regresión. Es la cifra principal.`) +}; + +const zh_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`所有隐藏检查都通过且没有出现回归的任务占比,按难度加权。这是最核心的数字。`) +}; + +const ja_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`すべての非公開チェックに合格し、リグレッションも出なかったタスクの割合を、難易度で重み付けした値です。これが中心となる数値です。`) +}; + +const ko_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`모든 비공개 검사를 통과하고 회귀가 없었던 태스크의 비율을 난이도로 가중한 값입니다. 이 페이지의 대표 수치입니다.`) +}; + +const zh_hant1_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`所有隱藏檢查都通過且沒有出現迴歸的任務占比,按難度加權。這是最核心的數字。`) +}; + +const de_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Nach Schwierigkeit gewichteter Anteil der Aufgaben, bei denen jede verborgene Prüfung bestanden wurde und keine Regression auftrat. Das ist die zentrale Kennzahl.`) +}; + +const fr_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Part des tâches, pondérée par la difficulté, où toutes les vérifications cachées sont passées et où aucune régression n'est apparue. C'est le chiffre principal.`) +}; + +const uk_fixproofdefresolvedindex3 = /** @type {(inputs: Fixproofdefresolvedindex3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Зважена за складністю частка задач, де пройшли всі приховані перевірки й не виникло регресій. Це головне число.`) +}; + +/** +* | output | +* | --- | +* | "Difficulty-weighted share of tasks where every hidden check passed and no regression appeared. This is the headline number." | +* +* @param {Fixproofdefresolvedindex3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefresolvedindex3 = /** @type {((inputs?: Fixproofdefresolvedindex3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefresolvedindex3(inputs) + if (locale === "zh") return zh_fixproofdefresolvedindex3(inputs) + if (locale === "ja") return ja_fixproofdefresolvedindex3(inputs) + if (locale === "ko") return ko_fixproofdefresolvedindex3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefresolvedindex3(inputs) + if (locale === "de") return de_fixproofdefresolvedindex3(inputs) + if (locale === "fr") return fr_fixproofdefresolvedindex3(inputs) + if (locale === "uk") return uk_fixproofdefresolvedindex3(inputs) + return en_fixproofdefresolvedindex3(inputs) +}); +export { fixproofdefresolvedindex3 as "fixproofDefResolvedIndex" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdefsolvedovergraded4.js b/apps/web/src/paraglide/messages/fixproofdefsolvedovergraded4.js new file mode 100644 index 000000000..e22089424 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdefsolvedovergraded4.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdefsolvedovergraded4Inputs */ + +const en_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tasks fully resolved out of the tasks graded so far. Pending tasks and excluded runs are not in the denominator.`) +}; + +const es_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tareas resueltas por completo sobre las tareas evaluadas hasta ahora. Las tareas pendientes y las ejecuciones excluidas no están en el denominador.`) +}; + +const zh_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`在目前已评测的任务中完全解决的数量。待运行的任务和被排除的运行不计入分母。`) +}; + +const ja_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`これまでに採点したタスクのうち、完全に解決したタスクの数です。保留中のタスクと除外した実行は分母に含みません。`) +}; + +const ko_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`지금까지 채점한 태스크 중 완전히 해결한 태스크입니다. 대기 중인 태스크와 제외된 실행은 분모에 넣지 않습니다.`) +}; + +const zh_hant1_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`在目前已評測的任務中完全解決的數量。待執行的任務和被排除的執行不計入分母。`) +}; + +const de_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Vollständig gelöste Aufgaben von den bisher bewerteten Aufgaben. Ausstehende Aufgaben und ausgeschlossene Läufe stehen nicht im Nenner.`) +}; + +const fr_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tâches entièrement résolues sur les tâches évaluées jusqu'ici. Les tâches en attente et les exécutions exclues ne sont pas au dénominateur.`) +}; + +const uk_fixproofdefsolvedovergraded4 = /** @type {(inputs: Fixproofdefsolvedovergraded4Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Повністю вирішені задачі з тих, що вже оцінені. Задачі в очікуванні та виключені запуски не входять у знаменник.`) +}; + +/** +* | output | +* | --- | +* | "Tasks fully resolved out of the tasks graded so far. Pending tasks and excluded runs are not in the denominator." | +* +* @param {Fixproofdefsolvedovergraded4Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdefsolvedovergraded4 = /** @type {((inputs?: Fixproofdefsolvedovergraded4Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdefsolvedovergraded4(inputs) + if (locale === "zh") return zh_fixproofdefsolvedovergraded4(inputs) + if (locale === "ja") return ja_fixproofdefsolvedovergraded4(inputs) + if (locale === "ko") return ko_fixproofdefsolvedovergraded4(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdefsolvedovergraded4(inputs) + if (locale === "de") return de_fixproofdefsolvedovergraded4(inputs) + if (locale === "fr") return fr_fixproofdefsolvedovergraded4(inputs) + if (locale === "uk") return uk_fixproofdefsolvedovergraded4(inputs) + return en_fixproofdefsolvedovergraded4(inputs) +}); +export { fixproofdefsolvedovergraded4 as "fixproofDefSolvedOverGraded" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdeftestedits3.js b/apps/web/src/paraglide/messages/fixproofdeftestedits3.js new file mode 100644 index 000000000..63bdb1e5f --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdeftestedits3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdeftestedits3Inputs */ + +const en_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Edits the agent made to test files. The harness reverts them before grading.`) +}; + +const es_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Cambios que el agente hizo en archivos de prueba. El harness los revierte antes de evaluar.`) +}; + +const zh_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`代理对测试文件所做的改动。harness 会在评测前把它们还原。`) +}; + +const ja_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`エージェントがテストファイルに加えた変更です。ハーネスが採点前に差し戻します。`) +}; + +const ko_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`에이전트가 테스트 파일에 가한 수정입니다. 하네스가 채점 전에 되돌립니다.`) +}; + +const zh_hant1_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`代理程式對測試檔案所做的改動。harness 會在評測前把它們還原。`) +}; + +const de_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Änderungen, die der Agent an Testdateien vorgenommen hat. Das Harness setzt sie vor der Bewertung zurück.`) +}; + +const fr_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Modifications que l'agent a apportées aux fichiers de test. Le harness les annule avant l'évaluation.`) +}; + +const uk_fixproofdeftestedits3 = /** @type {(inputs: Fixproofdeftestedits3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Зміни, які агент вніс у файли тестів. Harness скасовує їх перед оцінюванням.`) +}; + +/** +* | output | +* | --- | +* | "Edits the agent made to test files. The harness reverts them before grading." | +* +* @param {Fixproofdeftestedits3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdeftestedits3 = /** @type {((inputs?: Fixproofdeftestedits3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdeftestedits3(inputs) + if (locale === "zh") return zh_fixproofdeftestedits3(inputs) + if (locale === "ja") return ja_fixproofdeftestedits3(inputs) + if (locale === "ko") return ko_fixproofdeftestedits3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdeftestedits3(inputs) + if (locale === "de") return de_fixproofdeftestedits3(inputs) + if (locale === "fr") return fr_fixproofdeftestedits3(inputs) + if (locale === "uk") return uk_fixproofdeftestedits3(inputs) + return en_fixproofdeftestedits3(inputs) +}); +export { fixproofdeftestedits3 as "fixproofDefTestEdits" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofdeftrials2.js b/apps/web/src/paraglide/messages/fixproofdeftrials2.js new file mode 100644 index 000000000..cbda1959d --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofdeftrials2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofdeftrials2Inputs */ + +const en_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Runs per task. One trial is a single sample, so read small differences as noise.`) +}; + +const es_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Ejecuciones por tarea. Un intento es una sola muestra, así que lee las diferencias pequeñas como ruido.`) +}; + +const zh_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每个任务的运行次数。一次试验只是一个样本,因此细小的差距应当视为噪声。`) +}; + +const ja_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`タスクあたりの実行回数です。1 回の試行はサンプル 1 つなので、小さな差はノイズとして読んでください。`) +}; + +const ko_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`태스크당 실행 횟수입니다. 한 번의 시도는 표본 하나이므로 작은 차이는 잡음으로 보세요.`) +}; + +const zh_hant1_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每個任務的執行次數。一次試驗只是一個樣本,因此細小的差距應當視為雜訊。`) +}; + +const de_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Läufe pro Aufgabe. Ein Versuch ist eine einzelne Stichprobe, kleine Unterschiede sind also Rauschen.`) +}; + +const fr_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Exécutions par tâche. Un essai est un échantillon unique : lisez les petits écarts comme du bruit.`) +}; + +const uk_fixproofdeftrials2 = /** @type {(inputs: Fixproofdeftrials2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Запусків на задачу. Одна спроба є однією вибіркою, тому невеликі відмінності варто читати як шум.`) +}; + +/** +* | output | +* | --- | +* | "Runs per task. One trial is a single sample, so read small differences as noise." | +* +* @param {Fixproofdeftrials2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofdeftrials2 = /** @type {((inputs?: Fixproofdeftrials2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofdeftrials2(inputs) + if (locale === "zh") return zh_fixproofdeftrials2(inputs) + if (locale === "ja") return ja_fixproofdeftrials2(inputs) + if (locale === "ko") return ko_fixproofdeftrials2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofdeftrials2(inputs) + if (locale === "de") return de_fixproofdeftrials2(inputs) + if (locale === "fr") return fr_fixproofdeftrials2(inputs) + if (locale === "uk") return uk_fixproofdeftrials2(inputs) + return en_fixproofdeftrials2(inputs) +}); +export { fixproofdeftrials2 as "fixproofDefTrials" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofgradedoftotal3.js b/apps/web/src/paraglide/messages/fixproofgradedoftotal3.js new file mode 100644 index 000000000..5873d37f3 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofgradedoftotal3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{ graded: NonNullable, total: NonNullable }} Fixproofgradedoftotal3Inputs */ + +const en_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.graded} of ${i?.total}`) +}; + +const es_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.graded} de ${i?.total}`) +}; + +const zh_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.total} 个中的 ${i?.graded} 个`) +}; + +const ja_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.total} 件中 ${i?.graded} 件`) +}; + +const ko_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.total}개 중 ${i?.graded}개`) +}; + +const zh_hant1_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.total} 個中的 ${i?.graded} 個`) +}; + +const de_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.graded} von ${i?.total}`) +}; + +const fr_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.graded} sur ${i?.total}`) +}; + +const uk_fixproofgradedoftotal3 = /** @type {(inputs: Fixproofgradedoftotal3Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.graded} з ${i?.total}`) +}; + +/** +* | output | +* | --- | +* | "{graded} of {total}" | +* +* @param {Fixproofgradedoftotal3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofgradedoftotal3 = /** @type {((inputs: Fixproofgradedoftotal3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofgradedoftotal3(inputs) + if (locale === "zh") return zh_fixproofgradedoftotal3(inputs) + if (locale === "ja") return ja_fixproofgradedoftotal3(inputs) + if (locale === "ko") return ko_fixproofgradedoftotal3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofgradedoftotal3(inputs) + if (locale === "de") return de_fixproofgradedoftotal3(inputs) + if (locale === "fr") return fr_fixproofgradedoftotal3(inputs) + if (locale === "uk") return uk_fixproofgradedoftotal3(inputs) + return en_fixproofgradedoftotal3(inputs) +}); +export { fixproofgradedoftotal3 as "fixproofGradedOfTotal" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofprovenancesummary2.js b/apps/web/src/paraglide/messages/fixproofprovenancesummary2.js new file mode 100644 index 000000000..9349e9063 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofprovenancesummary2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofprovenancesummary2Inputs */ + +const en_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Every number comes from a recorded unattended run of the named agent CLI on a dedicated Linux bench machine, against the task's base commit and graded by hidden tests that were proven red at that commit and green with the maintainers' fix. Nothing here is hand-scored.`) +}; + +const es_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Cada cifra sale de una ejecución registrada y sin supervisión de la CLI del agente indicada, en una máquina de pruebas Linux dedicada, contra el commit base de la tarea y puntuada por pruebas ocultas que estaban en rojo en ese commit y en verde con la corrección de los mantenedores. Aquí no hay nada puntuado a mano.`) +}; + +const zh_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每个数字都来自在专用 Linux 基准机器上无人值守运行指定 agent CLI 的记录,针对任务的基础提交,由隐藏测试判定:这些测试在该提交上确认为红,在维护者的修复下确认为绿。这里没有任何人工打分。`) +}; + +const ja_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`すべての数値は、専用の Linux ベンチマシンで指定のエージェント CLI を無人実行した記録から得ています。判定はタスクのベースコミットに対して行い、そのコミットでは赤、メンテナの修正では緑になることを確認済みの非公開テストが採点します。手作業での採点はありません。`) +}; + +const ko_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`모든 수치는 전용 Linux 벤치 머신에서 지정된 에이전트 CLI를 무인으로 실행한 기록에서 나옵니다. 채점은 태스크의 기준 커밋을 대상으로 하며, 그 커밋에서는 실패하고 메인테이너의 수정으로는 통과함이 확인된 비공개 테스트가 판정합니다. 손으로 매긴 값은 없습니다.`) +}; + +const zh_hant1_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每個數字都來自在專用 Linux 基準機器上無人值守執行指定 agent CLI 的記錄,針對任務的基礎 commit,由隱藏測試判定:這些測試在該 commit 上確認為紅,在維護者的修正下確認為綠。這裡沒有任何人工評分。`) +}; + +const de_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Jede Zahl stammt aus einem aufgezeichneten, unbeaufsichtigten Lauf des genannten Agent-CLI auf einer eigenen Linux-Benchmaschine, gegen den Basis-Commit der Aufgabe und bewertet von verborgenen Tests, die bei diesem Commit nachweislich rot und mit dem Fix der Maintainer grün waren. Nichts hier ist von Hand bewertet.`) +}; + +const fr_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Chaque chiffre sort d'une exécution enregistrée et sans supervision du CLI d'agent indiqué, sur une machine de test Linux dédiée, face au commit de base de la tâche et noté par des tests cachés vérifiés rouges à ce commit et verts avec le correctif des mainteneurs. Rien ici n'est noté à la main.`) +}; + +const uk_fixproofprovenancesummary2 = /** @type {(inputs: Fixproofprovenancesummary2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Кожне число походить із записаного автономного запуску вказаного агентського CLI на окремій Linux-машині для бенчмарків, проти базового коміту задачі та з оцінюванням прихованими тестами, які були червоними на цьому коміті й зеленими з виправленням мейнтейнерів. Тут немає нічого оціненого вручну.`) +}; + +/** +* | output | +* | --- | +* | "Every number comes from a recorded unattended run of the named agent CLI on a dedicated Linux bench machine, against the task's base commit and graded by hid..." | +* +* @param {Fixproofprovenancesummary2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofprovenancesummary2 = /** @type {((inputs?: Fixproofprovenancesummary2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofprovenancesummary2(inputs) + if (locale === "zh") return zh_fixproofprovenancesummary2(inputs) + if (locale === "ja") return ja_fixproofprovenancesummary2(inputs) + if (locale === "ko") return ko_fixproofprovenancesummary2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofprovenancesummary2(inputs) + if (locale === "de") return de_fixproofprovenancesummary2(inputs) + if (locale === "fr") return fr_fixproofprovenancesummary2(inputs) + if (locale === "uk") return uk_fixproofprovenancesummary2(inputs) + return en_fixproofprovenancesummary2(inputs) +}); +export { fixproofprovenancesummary2 as "fixproofProvenanceSummary" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofseotitle2.js b/apps/web/src/paraglide/messages/fixproofseotitle2.js new file mode 100644 index 000000000..3ad371a85 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofseotitle2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofseotitle2Inputs */ + +const en_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof: sealed coding-agent benchmark`) +}; + +const es_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof: benchmark sellado para agentes de programación`) +}; + +const zh_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof:封闭式编程代理基准测试`) +}; + +const ja_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof: 封印されたコーディングエージェントのベンチマーク`) +}; + +const ko_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof: 봉인된 코딩 에이전트 벤치마크`) +}; + +const zh_hant1_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof:封閉式程式代理基準測試`) +}; + +const de_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof: versiegelter Benchmark für Coding-Agenten`) +}; + +const fr_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof : benchmark scellé pour agents de codage`) +}; + +const uk_fixproofseotitle2 = /** @type {(inputs: Fixproofseotitle2Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fixproof: закритий бенчмарк для агентів для коду`) +}; + +/** +* | output | +* | --- | +* | "Fixproof: sealed coding-agent benchmark" | +* +* @param {Fixproofseotitle2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofseotitle2 = /** @type {((inputs?: Fixproofseotitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofseotitle2(inputs) + if (locale === "zh") return zh_fixproofseotitle2(inputs) + if (locale === "ja") return ja_fixproofseotitle2(inputs) + if (locale === "ko") return ko_fixproofseotitle2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofseotitle2(inputs) + if (locale === "de") return de_fixproofseotitle2(inputs) + if (locale === "fr") return fr_fixproofseotitle2(inputs) + if (locale === "uk") return uk_fixproofseotitle2(inputs) + return en_fixproofseotitle2(inputs) +}); +export { fixproofseotitle2 as "fixproofSeoTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofsortaria2.js b/apps/web/src/paraglide/messages/fixproofsortaria2.js new file mode 100644 index 000000000..20f5784f8 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofsortaria2.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{ column: NonNullable }} Fixproofsortaria2Inputs */ + +const en_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Sort by ${i?.column}`) +}; + +const es_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Ordenar por ${i?.column}`) +}; + +const zh_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`按 ${i?.column} 排序`) +}; + +const ja_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.column} で並べ替え`) +}; + +const ko_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`${i?.column} 기준 정렬`) +}; + +const zh_hant1_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`按 ${i?.column} 排序`) +}; + +const de_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Nach ${i?.column} sortieren`) +}; + +const fr_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Trier par ${i?.column}`) +}; + +const uk_fixproofsortaria2 = /** @type {(inputs: Fixproofsortaria2Inputs) => LocalizedString} */ (i) => { + return /** @type {LocalizedString} */ (`Сортувати за ${i?.column}`) +}; + +/** +* | output | +* | --- | +* | "Sort by {column}" | +* +* @param {Fixproofsortaria2Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofsortaria2 = /** @type {((inputs: Fixproofsortaria2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofsortaria2(inputs) + if (locale === "zh") return zh_fixproofsortaria2(inputs) + if (locale === "ja") return ja_fixproofsortaria2(inputs) + if (locale === "ko") return ko_fixproofsortaria2(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofsortaria2(inputs) + if (locale === "de") return de_fixproofsortaria2(inputs) + if (locale === "fr") return fr_fixproofsortaria2(inputs) + if (locale === "uk") return uk_fixproofsortaria2(inputs) + return en_fixproofsortaria2(inputs) +}); +export { fixproofsortaria2 as "fixproofSortAria" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofstatusdatelabel3.js b/apps/web/src/paraglide/messages/fixproofstatusdatelabel3.js new file mode 100644 index 000000000..8a7bf3f87 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofstatusdatelabel3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofstatusdatelabel3Inputs */ + +const en_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Date`) +}; + +const es_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Fecha`) +}; + +const zh_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`日期`) +}; + +const ja_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`日付`) +}; + +const ko_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`날짜`) +}; + +const zh_hant1_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`日期`) +}; + +const de_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Datum`) +}; + +const fr_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Date`) +}; + +const uk_fixproofstatusdatelabel3 = /** @type {(inputs: Fixproofstatusdatelabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Дата`) +}; + +/** +* | output | +* | --- | +* | "Date" | +* +* @param {Fixproofstatusdatelabel3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofstatusdatelabel3 = /** @type {((inputs?: Fixproofstatusdatelabel3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofstatusdatelabel3(inputs) + if (locale === "zh") return zh_fixproofstatusdatelabel3(inputs) + if (locale === "ja") return ja_fixproofstatusdatelabel3(inputs) + if (locale === "ko") return ko_fixproofstatusdatelabel3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofstatusdatelabel3(inputs) + if (locale === "de") return de_fixproofstatusdatelabel3(inputs) + if (locale === "fr") return fr_fixproofstatusdatelabel3(inputs) + if (locale === "uk") return uk_fixproofstatusdatelabel3(inputs) + return en_fixproofstatusdatelabel3(inputs) +}); +export { fixproofstatusdatelabel3 as "fixproofStatusDateLabel" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofstatusgradedlabel3.js b/apps/web/src/paraglide/messages/fixproofstatusgradedlabel3.js new file mode 100644 index 000000000..b2087f3c7 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofstatusgradedlabel3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofstatusgradedlabel3Inputs */ + +const en_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tasks graded`) +}; + +const es_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tareas evaluadas`) +}; + +const zh_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`已评测任务`) +}; + +const ja_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`採点済みタスク`) +}; + +const ko_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`채점한 태스크`) +}; + +const zh_hant1_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`已評測任務`) +}; + +const de_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Bewertete Aufgaben`) +}; + +const fr_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Tâches évaluées`) +}; + +const uk_fixproofstatusgradedlabel3 = /** @type {(inputs: Fixproofstatusgradedlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Оцінено задач`) +}; + +/** +* | output | +* | --- | +* | "Tasks graded" | +* +* @param {Fixproofstatusgradedlabel3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofstatusgradedlabel3 = /** @type {((inputs?: Fixproofstatusgradedlabel3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofstatusgradedlabel3(inputs) + if (locale === "zh") return zh_fixproofstatusgradedlabel3(inputs) + if (locale === "ja") return ja_fixproofstatusgradedlabel3(inputs) + if (locale === "ko") return ko_fixproofstatusgradedlabel3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofstatusgradedlabel3(inputs) + if (locale === "de") return de_fixproofstatusgradedlabel3(inputs) + if (locale === "fr") return fr_fixproofstatusgradedlabel3(inputs) + if (locale === "uk") return uk_fixproofstatusgradedlabel3(inputs) + return en_fixproofstatusgradedlabel3(inputs) +}); +export { fixproofstatusgradedlabel3 as "fixproofStatusGradedLabel" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofstatusrunlabel3.js b/apps/web/src/paraglide/messages/fixproofstatusrunlabel3.js new file mode 100644 index 000000000..e79c7e774 --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofstatusrunlabel3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofstatusrunlabel3Inputs */ + +const en_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Dry run`) +}; + +const es_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Ejecución de prueba`) +}; + +const zh_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`试运行`) +}; + +const ja_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`ドライラン`) +}; + +const ko_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`드라이런`) +}; + +const zh_hant1_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`試執行`) +}; + +const de_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Testlauf`) +}; + +const fr_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Essai à blanc`) +}; + +const uk_fixproofstatusrunlabel3 = /** @type {(inputs: Fixproofstatusrunlabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Пробний запуск`) +}; + +/** +* | output | +* | --- | +* | "Dry run" | +* +* @param {Fixproofstatusrunlabel3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofstatusrunlabel3 = /** @type {((inputs?: Fixproofstatusrunlabel3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofstatusrunlabel3(inputs) + if (locale === "zh") return zh_fixproofstatusrunlabel3(inputs) + if (locale === "ja") return ja_fixproofstatusrunlabel3(inputs) + if (locale === "ko") return ko_fixproofstatusrunlabel3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofstatusrunlabel3(inputs) + if (locale === "de") return de_fixproofstatusrunlabel3(inputs) + if (locale === "fr") return fr_fixproofstatusrunlabel3(inputs) + if (locale === "uk") return uk_fixproofstatusrunlabel3(inputs) + return en_fixproofstatusrunlabel3(inputs) +}); +export { fixproofstatusrunlabel3 as "fixproofStatusRunLabel" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/fixproofstatustrialslabel3.js b/apps/web/src/paraglide/messages/fixproofstatustrialslabel3.js new file mode 100644 index 000000000..91e36de1f --- /dev/null +++ b/apps/web/src/paraglide/messages/fixproofstatustrialslabel3.js @@ -0,0 +1,65 @@ +/* eslint-disable */ +import { getLocale, experimentalStaticLocale } from '../runtime.js'; + +/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ + +/** @typedef {{}} Fixproofstatustrialslabel3Inputs */ + +const en_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Trials per task`) +}; + +const es_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Intentos por tarea`) +}; + +const zh_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每个任务的试验次数`) +}; + +const ja_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`タスクあたりの試行数`) +}; + +const ko_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`태스크당 시도`) +}; + +const zh_hant1_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`每個任務的試驗次數`) +}; + +const de_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Versuche pro Aufgabe`) +}; + +const fr_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Essais par tâche`) +}; + +const uk_fixproofstatustrialslabel3 = /** @type {(inputs: Fixproofstatustrialslabel3Inputs) => LocalizedString} */ () => { + return /** @type {LocalizedString} */ (`Спроб на задачу`) +}; + +/** +* | output | +* | --- | +* | "Trials per task" | +* +* @param {Fixproofstatustrialslabel3Inputs} inputs +* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options +* @returns {LocalizedString} +*/ +const fixproofstatustrialslabel3 = /** @type {((inputs?: Fixproofstatustrialslabel3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { + const locale = experimentalStaticLocale ?? options.locale ?? getLocale() + if (locale === "es") return es_fixproofstatustrialslabel3(inputs) + if (locale === "zh") return zh_fixproofstatustrialslabel3(inputs) + if (locale === "ja") return ja_fixproofstatustrialslabel3(inputs) + if (locale === "ko") return ko_fixproofstatustrialslabel3(inputs) + if (locale === "zh-Hant") return zh_hant1_fixproofstatustrialslabel3(inputs) + if (locale === "de") return de_fixproofstatustrialslabel3(inputs) + if (locale === "fr") return fr_fixproofstatustrialslabel3(inputs) + if (locale === "uk") return uk_fixproofstatustrialslabel3(inputs) + return en_fixproofstatustrialslabel3(inputs) +}); +export { fixproofstatustrialslabel3 as "fixproofStatusTrialsLabel" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmagentdescription2.js b/apps/web/src/paraglide/messages/llmagentdescription2.js deleted file mode 100644 index 2705251f4..000000000 --- a/apps/web/src/paraglide/messages/llmagentdescription2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmagentdescription2Inputs */ - -const en_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`One MCP server, every spec-to-scaffold tool the benchmark used. Pick your agent, paste, done.`) -}; - -const es_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Un servidor MCP y todas las herramientas para pasar de especificación a scaffold que usó el benchmark. Elige tu agente, pega y listo.`) -}; - -const zh_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`一个 MCP 服务器,包含 benchmark 使用的所有 spec-to-scaffold 工具。选择代理,粘贴,就绪。`) -}; - -const ja_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`1 台の MCP サーバー、ベンチマークで使用されたすべての仕様から足場へのツール。エージェントを選択し、貼り付けて完了です。`) -}; - -const ko_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`하나의 MCP 서버, 벤치마크에서 사용된 모든 사양-스캐폴드 도구. 에이전트를 선택하고 붙여넣으면 완료됩니다.`) -}; - -const zh_hant1_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`一個 MCP 伺服器,包含 benchmark 使用的所有 spec-to-scaffold 工具。選擇代理,貼上,就緒。`) -}; - -const de_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ein MCP-Server, jedes vom Benchmark verwendete Spec-to-Scaffold-Tool. Wählen Sie Ihren Agenten aus, fügen Sie ihn ein, fertig.`) -}; - -const fr_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Un serveur MCP, chaque outil de spécification à échafaudage utilisé par le benchmark. Choisissez votre agent, collez, c'est fait.`) -}; - -const uk_llmagentdescription2 = /** @type {(inputs: Llmagentdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Один MCP-сервер і всі spec-to-scaffold інструменти, які використовував бенчмарк. Оберіть агента, вставте команду - готово.`) -}; - -/** -* | output | -* | --- | -* | "One MCP server, every spec-to-scaffold tool the benchmark used. Pick your agent, paste, done." | -* -* @param {Llmagentdescription2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmagentdescription2 = /** @type {((inputs?: Llmagentdescription2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmagentdescription2(inputs) - if (locale === "zh") return zh_llmagentdescription2(inputs) - if (locale === "ja") return ja_llmagentdescription2(inputs) - if (locale === "ko") return ko_llmagentdescription2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmagentdescription2(inputs) - if (locale === "de") return de_llmagentdescription2(inputs) - if (locale === "fr") return fr_llmagentdescription2(inputs) - if (locale === "uk") return uk_llmagentdescription2(inputs) - return en_llmagentdescription2(inputs) -}); -export { llmagentdescription2 as "llmAgentDescription" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmagenttitle2.js b/apps/web/src/paraglide/messages/llmagenttitle2.js deleted file mode 100644 index ddcbe7c34..000000000 --- a/apps/web/src/paraglide/messages/llmagenttitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmagenttitle2Inputs */ - -const en_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Give your agent the fast path.`) -}; - -const es_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Dale a tu agente la ruta rápida.`) -}; - -const zh_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`给你的代理一条快路径。`) -}; - -const ja_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`エージェントに高速パスを提供します。`) -}; - -const ko_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`에이전트에게 빠른 경로를 제공하세요.`) -}; - -const zh_hant1_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`給你的代理一條快路徑。`) -}; - -const de_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Geben Sie Ihrem Agenten den schnellen Weg.`) -}; - -const fr_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Donnez à votre agent la voie rapide.`) -}; - -const uk_llmagenttitle2 = /** @type {(inputs: Llmagenttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Дайте агенту короткий шлях.`) -}; - -/** -* | output | -* | --- | -* | "Give your agent the fast path." | -* -* @param {Llmagenttitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmagenttitle2 = /** @type {((inputs?: Llmagenttitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmagenttitle2(inputs) - if (locale === "zh") return zh_llmagenttitle2(inputs) - if (locale === "ja") return ja_llmagenttitle2(inputs) - if (locale === "ko") return ko_llmagenttitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmagenttitle2(inputs) - if (locale === "de") return de_llmagenttitle2(inputs) - if (locale === "fr") return fr_llmagenttitle2(inputs) - if (locale === "uk") return uk_llmagenttitle2(inputs) - return en_llmagenttitle2(inputs) -}); -export { llmagenttitle2 as "llmAgentTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmallsupportedclients3.js b/apps/web/src/paraglide/messages/llmallsupportedclients3.js deleted file mode 100644 index 39ba11c92..000000000 --- a/apps/web/src/paraglide/messages/llmallsupportedclients3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmallsupportedclients3Inputs */ - -const en_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`all supported clients`) -}; - -const es_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`todos los clientes soportados`) -}; - -const zh_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`所有支持的客户端`) -}; - -const ja_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`サポートされているすべてのクライアント`) -}; - -const ko_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`지원되는 모든 클라이언트`) -}; - -const zh_hant1_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`所有支援的客戶端`) -}; - -const de_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`alle unterstützten Clients`) -}; - -const fr_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`tous les clients pris en charge`) -}; - -const uk_llmallsupportedclients3 = /** @type {(inputs: Llmallsupportedclients3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`всі підтримувані клієнти`) -}; - -/** -* | output | -* | --- | -* | "all supported clients" | -* -* @param {Llmallsupportedclients3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmallsupportedclients3 = /** @type {((inputs?: Llmallsupportedclients3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmallsupportedclients3(inputs) - if (locale === "zh") return zh_llmallsupportedclients3(inputs) - if (locale === "ja") return ja_llmallsupportedclients3(inputs) - if (locale === "ko") return ko_llmallsupportedclients3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmallsupportedclients3(inputs) - if (locale === "de") return de_llmallsupportedclients3(inputs) - if (locale === "fr") return fr_llmallsupportedclients3(inputs) - if (locale === "uk") return uk_llmallsupportedclients3(inputs) - return en_llmallsupportedclients3(inputs) -}); -export { llmallsupportedclients3 as "llmAllSupportedClients" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmavgscaffoldtime3.js b/apps/web/src/paraglide/messages/llmavgscaffoldtime3.js deleted file mode 100644 index 78ffcb65e..000000000 --- a/apps/web/src/paraglide/messages/llmavgscaffoldtime3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmavgscaffoldtime3Inputs */ - -const en_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Avg scaffold time`) -}; - -const es_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tiempo medio de scaffold`) -}; - -const zh_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`平均 scaffold 时间`) -}; - -const ja_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`平均スキャフォールド時間`) -}; - -const ko_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`평균 스캐폴드 시간`) -}; - -const zh_hant1_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`平均 scaffold 時間`) -}; - -const de_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Durchschnittliche Gerüstzeit`) -}; - -const fr_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Temps moyen d'échafaudage`) -}; - -const uk_llmavgscaffoldtime3 = /** @type {(inputs: Llmavgscaffoldtime3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Середній час скафолдингу`) -}; - -/** -* | output | -* | --- | -* | "Avg scaffold time" | -* -* @param {Llmavgscaffoldtime3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmavgscaffoldtime3 = /** @type {((inputs?: Llmavgscaffoldtime3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmavgscaffoldtime3(inputs) - if (locale === "zh") return zh_llmavgscaffoldtime3(inputs) - if (locale === "ja") return ja_llmavgscaffoldtime3(inputs) - if (locale === "ko") return ko_llmavgscaffoldtime3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmavgscaffoldtime3(inputs) - if (locale === "de") return de_llmavgscaffoldtime3(inputs) - if (locale === "fr") return fr_llmavgscaffoldtime3(inputs) - if (locale === "uk") return uk_llmavgscaffoldtime3(inputs) - return en_llmavgscaffoldtime3(inputs) -}); -export { llmavgscaffoldtime3 as "llmAvgScaffoldTime" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmbenchmarkdescription2.js b/apps/web/src/paraglide/messages/llmbenchmarkdescription2.js deleted file mode 100644 index faa53713f..000000000 --- a/apps/web/src/paraglide/messages/llmbenchmarkdescription2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmbenchmarkdescription2Inputs */ - -const en_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Measuring coding agents on real fullstack scaffolding tasks - time, tokens, cost, and whether the result actually builds.`) -}; - -const es_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Mide agentes de programación en tareas reales de scaffolding fullstack: tiempo, tokens, coste y si el resultado realmente compila.`) -}; - -const zh_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`用真实全栈脚手架任务衡量编程代理:时间、tokens、成本,以及结果是否真的能构建。`) -}; - -const ja_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`実際のフルスタック スキャフォールディング タスクにおけるコーディング エージェントを測定します (時間、トークン、コスト、結果が実際にビルドされるかどうか)。`) -}; - -const ko_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`시간, 토큰, 비용 및 결과가 실제로 구축되는지 여부 등 실제 풀스택 스캐폴딩 작업에서 코딩 에이전트를 측정합니다.`) -}; - -const zh_hant1_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`用真實全端鷹架任務衡量程式設計代理:時間、tokens、成本,以及結果是否真的能建構。`) -}; - -const de_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Messung von Codierungsagenten bei echten Fullstack-Scaffolding-Aufgaben – Zeit, Token, Kosten und ob das Ergebnis tatsächlich erstellt wird.`) -}; - -const fr_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Mesurer les agents de codage sur de véritables tâches d'échafaudage fullstack : temps, jetons, coût et si le résultat se compile réellement.`) -}; - -const uk_llmbenchmarkdescription2 = /** @type {(inputs: Llmbenchmarkdescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Вимірюємо агентів для коду на реальних задачах фулстек-скафолдингу: час, токени, вартість і чи справді результат збирається.`) -}; - -/** -* | output | -* | --- | -* | "Measuring coding agents on real fullstack scaffolding tasks - time, tokens, cost, and whether the result actually builds." | -* -* @param {Llmbenchmarkdescription2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmbenchmarkdescription2 = /** @type {((inputs?: Llmbenchmarkdescription2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmbenchmarkdescription2(inputs) - if (locale === "zh") return zh_llmbenchmarkdescription2(inputs) - if (locale === "ja") return ja_llmbenchmarkdescription2(inputs) - if (locale === "ko") return ko_llmbenchmarkdescription2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmbenchmarkdescription2(inputs) - if (locale === "de") return de_llmbenchmarkdescription2(inputs) - if (locale === "fr") return fr_llmbenchmarkdescription2(inputs) - if (locale === "uk") return uk_llmbenchmarkdescription2(inputs) - return en_llmbenchmarkdescription2(inputs) -}); -export { llmbenchmarkdescription2 as "llmBenchmarkDescription" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmbenchmarkmetric2.js b/apps/web/src/paraglide/messages/llmbenchmarkmetric2.js deleted file mode 100644 index 1a6f52def..000000000 --- a/apps/web/src/paraglide/messages/llmbenchmarkmetric2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmbenchmarkmetric2Inputs */ - -const en_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark metric`) -}; - -const es_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Métrica del benchmark`) -}; - -const zh_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark 指标`) -}; - -const ja_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ベンチマーク指標`) -}; - -const ko_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`벤치마크 지표`) -}; - -const zh_hant1_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark 指標`) -}; - -const de_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark-Metrik`) -}; - -const fr_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Métrique de référence`) -}; - -const uk_llmbenchmarkmetric2 = /** @type {(inputs: Llmbenchmarkmetric2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Метрика бенчмарку`) -}; - -/** -* | output | -* | --- | -* | "Benchmark metric" | -* -* @param {Llmbenchmarkmetric2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmbenchmarkmetric2 = /** @type {((inputs?: Llmbenchmarkmetric2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmbenchmarkmetric2(inputs) - if (locale === "zh") return zh_llmbenchmarkmetric2(inputs) - if (locale === "ja") return ja_llmbenchmarkmetric2(inputs) - if (locale === "ko") return ko_llmbenchmarkmetric2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmbenchmarkmetric2(inputs) - if (locale === "de") return de_llmbenchmarkmetric2(inputs) - if (locale === "fr") return fr_llmbenchmarkmetric2(inputs) - if (locale === "uk") return uk_llmbenchmarkmetric2(inputs) - return en_llmbenchmarkmetric2(inputs) -}); -export { llmbenchmarkmetric2 as "llmBenchmarkMetric" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmbuildspassing2.js b/apps/web/src/paraglide/messages/llmbuildspassing2.js deleted file mode 100644 index f51df0a4b..000000000 --- a/apps/web/src/paraglide/messages/llmbuildspassing2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmbuildspassing2Inputs */ - -const en_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Builds passing`) -}; - -const es_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Builds que pasan`) -}; - -const zh_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`通过构建`) -}; - -const ja_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ビルドの合格`) -}; - -const ko_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`빌드 통과`) -}; - -const zh_hant1_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`透過建構`) -}; - -const de_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Erfolgreiche Builds`) -}; - -const fr_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Constructions réussies`) -}; - -const uk_llmbuildspassing2 = /** @type {(inputs: Llmbuildspassing2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Успішні збірки`) -}; - -/** -* | output | -* | --- | -* | "Builds passing" | -* -* @param {Llmbuildspassing2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmbuildspassing2 = /** @type {((inputs?: Llmbuildspassing2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmbuildspassing2(inputs) - if (locale === "zh") return zh_llmbuildspassing2(inputs) - if (locale === "ja") return ja_llmbuildspassing2(inputs) - if (locale === "ko") return ko_llmbuildspassing2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmbuildspassing2(inputs) - if (locale === "de") return de_llmbuildspassing2(inputs) - if (locale === "fr") return fr_llmbuildspassing2(inputs) - if (locale === "uk") return uk_llmbuildspassing2(inputs) - return en_llmbuildspassing2(inputs) -}); -export { llmbuildspassing2 as "llmBuildsPassing" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmclaudesweep2.js b/apps/web/src/paraglide/messages/llmclaudesweep2.js deleted file mode 100644 index 86b152cfc..000000000 --- a/apps/web/src/paraglide/messages/llmclaudesweep2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmclaudesweep2Inputs */ - -const en_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Jun 12 sweep`) -}; - -const es_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Barrido del 12 jun`) -}; - -const zh_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6 月 12 日批测`) -}; - -const ja_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6月12日のスイープ`) -}; - -const ko_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6월 12일 스윕`) -}; - -const zh_hant1_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6 月 12 日批測`) -}; - -const de_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`12. Juni Sweep`) -}; - -const fr_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Balayage du 12 juin`) -}; - -const uk_llmclaudesweep2 = /** @type {(inputs: Llmclaudesweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`прогін 12 червня`) -}; - -/** -* | output | -* | --- | -* | "Jun 12 sweep" | -* -* @param {Llmclaudesweep2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmclaudesweep2 = /** @type {((inputs?: Llmclaudesweep2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmclaudesweep2(inputs) - if (locale === "zh") return zh_llmclaudesweep2(inputs) - if (locale === "ja") return ja_llmclaudesweep2(inputs) - if (locale === "ko") return ko_llmclaudesweep2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmclaudesweep2(inputs) - if (locale === "de") return de_llmclaudesweep2(inputs) - if (locale === "fr") return fr_llmclaudesweep2(inputs) - if (locale === "uk") return uk_llmclaudesweep2(inputs) - return en_llmclaudesweep2(inputs) -}); -export { llmclaudesweep2 as "llmClaudeSweep" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmcodexsweep2.js b/apps/web/src/paraglide/messages/llmcodexsweep2.js deleted file mode 100644 index e742a5ced..000000000 --- a/apps/web/src/paraglide/messages/llmcodexsweep2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmcodexsweep2Inputs */ - -const en_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Jun 10 sweep`) -}; - -const es_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Barrido del 10 jun`) -}; - -const zh_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6 月 10 日批测`) -}; - -const ja_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6月10日のスイープ`) -}; - -const ko_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6월 10일 스윕`) -}; - -const zh_hant1_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6 月 10 日批測`) -}; - -const de_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`10. Juni Sweep`) -}; - -const fr_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Balayage du 10 juin`) -}; - -const uk_llmcodexsweep2 = /** @type {(inputs: Llmcodexsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`прогін 10 червня`) -}; - -/** -* | output | -* | --- | -* | "Jun 10 sweep" | -* -* @param {Llmcodexsweep2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmcodexsweep2 = /** @type {((inputs?: Llmcodexsweep2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmcodexsweep2(inputs) - if (locale === "zh") return zh_llmcodexsweep2(inputs) - if (locale === "ja") return ja_llmcodexsweep2(inputs) - if (locale === "ko") return ko_llmcodexsweep2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmcodexsweep2(inputs) - if (locale === "de") return de_llmcodexsweep2(inputs) - if (locale === "fr") return fr_llmcodexsweep2(inputs) - if (locale === "uk") return uk_llmcodexsweep2(inputs) - return en_llmcodexsweep2(inputs) -}); -export { llmcodexsweep2 as "llmCodexSweep" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmerrorrate2.js b/apps/web/src/paraglide/messages/llmerrorrate2.js deleted file mode 100644 index b5d2d9068..000000000 --- a/apps/web/src/paraglide/messages/llmerrorrate2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmerrorrate2Inputs */ - -const en_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Error rate`) -}; - -const es_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tasa de error`) -}; - -const zh_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`错误率`) -}; - -const ja_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`エラー率`) -}; - -const ko_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`오류율`) -}; - -const zh_hant1_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`錯誤率`) -}; - -const de_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Fehlerquote`) -}; - -const fr_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Taux d'erreur`) -}; - -const uk_llmerrorrate2 = /** @type {(inputs: Llmerrorrate2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Частота помилок`) -}; - -/** -* | output | -* | --- | -* | "Error rate" | -* -* @param {Llmerrorrate2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmerrorrate2 = /** @type {((inputs?: Llmerrorrate2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmerrorrate2(inputs) - if (locale === "zh") return zh_llmerrorrate2(inputs) - if (locale === "ja") return ja_llmerrorrate2(inputs) - if (locale === "ko") return ko_llmerrorrate2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmerrorrate2(inputs) - if (locale === "de") return de_llmerrorrate2(inputs) - if (locale === "fr") return fr_llmerrorrate2(inputs) - if (locale === "uk") return uk_llmerrorrate2(inputs) - return en_llmerrorrate2(inputs) -}); -export { llmerrorrate2 as "llmErrorRate" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmfailedbuilds2.js b/apps/web/src/paraglide/messages/llmfailedbuilds2.js deleted file mode 100644 index 4979f4385..000000000 --- a/apps/web/src/paraglide/messages/llmfailedbuilds2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmfailedbuilds2Inputs */ - -const en_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Failed builds`) -}; - -const es_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Builds fallidos`) -}; - -const zh_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`失败构建`) -}; - -const ja_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`失敗したビルド`) -}; - -const ko_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`실패한 빌드`) -}; - -const zh_hant1_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`失敗建構`) -}; - -const de_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Fehlgeschlagene Builds`) -}; - -const fr_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Constructions échouées`) -}; - -const uk_llmfailedbuilds2 = /** @type {(inputs: Llmfailedbuilds2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Невдалі збірки`) -}; - -/** -* | output | -* | --- | -* | "Failed builds" | -* -* @param {Llmfailedbuilds2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmfailedbuilds2 = /** @type {((inputs?: Llmfailedbuilds2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmfailedbuilds2(inputs) - if (locale === "zh") return zh_llmfailedbuilds2(inputs) - if (locale === "ja") return ja_llmfailedbuilds2(inputs) - if (locale === "ko") return ko_llmfailedbuilds2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmfailedbuilds2(inputs) - if (locale === "de") return de_llmfailedbuilds2(inputs) - if (locale === "fr") return fr_llmfailedbuilds2(inputs) - if (locale === "uk") return uk_llmfailedbuilds2(inputs) - return en_llmfailedbuilds2(inputs) -}); -export { llmfailedbuilds2 as "llmFailedBuilds" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmfastreliable2.js b/apps/web/src/paraglide/messages/llmfastreliable2.js deleted file mode 100644 index dd6ba5428..000000000 --- a/apps/web/src/paraglide/messages/llmfastreliable2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmfastreliable2Inputs */ - -const en_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`fast + reliable ↗`) -}; - -const es_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`rápido + fiable ↗`) -}; - -const zh_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`快速 + 可靠 ↗`) -}; - -const ja_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`高速 + 信頼性の高い ↗`) -}; - -const ko_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`빠르고 안정적 ​​↗`) -}; - -const zh_hant1_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`快 + 可靠 ↗`) -}; - -const de_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`schnell + zuverlässig ↗`) -}; - -const fr_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`rapide + fiable ↗`) -}; - -const uk_llmfastreliable2 = /** @type {(inputs: Llmfastreliable2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`швидко + надійно ↗`) -}; - -/** -* | output | -* | --- | -* | "fast + reliable ↗" | -* -* @param {Llmfastreliable2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmfastreliable2 = /** @type {((inputs?: Llmfastreliable2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmfastreliable2(inputs) - if (locale === "zh") return zh_llmfastreliable2(inputs) - if (locale === "ja") return ja_llmfastreliable2(inputs) - if (locale === "ko") return ko_llmfastreliable2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmfastreliable2(inputs) - if (locale === "de") return de_llmfastreliable2(inputs) - if (locale === "fr") return fr_llmfastreliable2(inputs) - if (locale === "uk") return uk_llmfastreliable2(inputs) - return en_llmfastreliable2(inputs) -}); -export { llmfastreliable2 as "llmFastReliable" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmfiltermodels2.js b/apps/web/src/paraglide/messages/llmfiltermodels2.js deleted file mode 100644 index e85e1ff4c..000000000 --- a/apps/web/src/paraglide/messages/llmfiltermodels2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmfiltermodels2Inputs */ - -const en_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Filter models`) -}; - -const es_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Filtrar modelos`) -}; - -const zh_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`筛选模型`) -}; - -const ja_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`フィルターモデル`) -}; - -const ko_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`모델 필터링`) -}; - -const zh_hant1_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`篩選模型`) -}; - -const de_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Modelle filtern`) -}; - -const fr_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Filtrer les modèles`) -}; - -const uk_llmfiltermodels2 = /** @type {(inputs: Llmfiltermodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Фільтрувати моделі`) -}; - -/** -* | output | -* | --- | -* | "Filter models" | -* -* @param {Llmfiltermodels2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmfiltermodels2 = /** @type {((inputs?: Llmfiltermodels2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmfiltermodels2(inputs) - if (locale === "zh") return zh_llmfiltermodels2(inputs) - if (locale === "ja") return ja_llmfiltermodels2(inputs) - if (locale === "ko") return ko_llmfiltermodels2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmfiltermodels2(inputs) - if (locale === "de") return de_llmfiltermodels2(inputs) - if (locale === "fr") return fr_llmfiltermodels2(inputs) - if (locale === "uk") return uk_llmfiltermodels2(inputs) - return en_llmfiltermodels2(inputs) -}); -export { llmfiltermodels2 as "llmFilterModels" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmlightsweep2.js b/apps/web/src/paraglide/messages/llmlightsweep2.js deleted file mode 100644 index c90687117..000000000 --- a/apps/web/src/paraglide/messages/llmlightsweep2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmlightsweep2Inputs */ - -const en_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Jun 12 light sweep`) -}; - -const es_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Barrido ligero del 12 jun`) -}; - -const zh_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6 月 12 日轻量批测`) -}; - -const ja_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6月12日 ライトスイープ`) -}; - -const ko_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6월 12일 라이트 스윕`) -}; - -const zh_hant1_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`6 月 12 日輕量批測`) -}; - -const de_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`12. Juni leichter Sweep`) -}; - -const fr_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Balayage léger du 12 juin`) -}; - -const uk_llmlightsweep2 = /** @type {(inputs: Llmlightsweep2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`легкий прогін 12 червня`) -}; - -/** -* | output | -* | --- | -* | "Jun 12 light sweep" | -* -* @param {Llmlightsweep2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmlightsweep2 = /** @type {((inputs?: Llmlightsweep2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmlightsweep2(inputs) - if (locale === "zh") return zh_llmlightsweep2(inputs) - if (locale === "ja") return ja_llmlightsweep2(inputs) - if (locale === "ko") return ko_llmlightsweep2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmlightsweep2(inputs) - if (locale === "de") return de_llmlightsweep2(inputs) - if (locale === "fr") return fr_llmlightsweep2(inputs) - if (locale === "uk") return uk_llmlightsweep2(inputs) - return en_llmlightsweep2(inputs) -}); -export { llmlightsweep2 as "llmLightSweep" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmmodels1.js b/apps/web/src/paraglide/messages/llmmodels1.js deleted file mode 100644 index 25982922d..000000000 --- a/apps/web/src/paraglide/messages/llmmodels1.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmmodels1Inputs */ - -const en_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Models`) -}; - -const es_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Modelos`) -}; - -const zh_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`模型`) -}; - -const ja_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`モデル`) -}; - -const ko_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`모델`) -}; - -const zh_hant1_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`模型`) -}; - -const de_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Modelle`) -}; - -const fr_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Modèles`) -}; - -const uk_llmmodels1 = /** @type {(inputs: Llmmodels1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Моделі`) -}; - -/** -* | output | -* | --- | -* | "Models" | -* -* @param {Llmmodels1Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmmodels1 = /** @type {((inputs?: Llmmodels1Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmmodels1(inputs) - if (locale === "zh") return zh_llmmodels1(inputs) - if (locale === "ja") return ja_llmmodels1(inputs) - if (locale === "ko") return ko_llmmodels1(inputs) - if (locale === "zh-Hant") return zh_hant1_llmmodels1(inputs) - if (locale === "de") return de_llmmodels1(inputs) - if (locale === "fr") return fr_llmmodels1(inputs) - if (locale === "uk") return uk_llmmodels1(inputs) - return en_llmmodels1(inputs) -}); -export { llmmodels1 as "llmModels" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmmostefficient2.js b/apps/web/src/paraglide/messages/llmmostefficient2.js deleted file mode 100644 index 60054292e..000000000 --- a/apps/web/src/paraglide/messages/llmmostefficient2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmmostefficient2Inputs */ - -const en_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`most efficient ↗`) -}; - -const es_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`más eficiente ↗`) -}; - -const zh_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`最高效 ↗`) -}; - -const ja_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`最も効率的 ↗`) -}; - -const ko_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`가장 효율적 ↗`) -}; - -const zh_hant1_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`最高效 ↗`) -}; - -const de_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`am effizientesten ↗`) -}; - -const fr_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`le plus efficace ↗`) -}; - -const uk_llmmostefficient2 = /** @type {(inputs: Llmmostefficient2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`найефективніший ↗`) -}; - -/** -* | output | -* | --- | -* | "most efficient ↗" | -* -* @param {Llmmostefficient2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmmostefficient2 = /** @type {((inputs?: Llmmostefficient2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmmostefficient2(inputs) - if (locale === "zh") return zh_llmmostefficient2(inputs) - if (locale === "ja") return ja_llmmostefficient2(inputs) - if (locale === "ko") return ko_llmmostefficient2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmmostefficient2(inputs) - if (locale === "de") return de_llmmostefficient2(inputs) - if (locale === "fr") return fr_llmmostefficient2(inputs) - if (locale === "uk") return uk_llmmostefficient2(inputs) - return en_llmmostefficient2(inputs) -}); -export { llmmostefficient2 as "llmMostEfficient" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmoutputtokens2.js b/apps/web/src/paraglide/messages/llmoutputtokens2.js deleted file mode 100644 index f58a378e6..000000000 --- a/apps/web/src/paraglide/messages/llmoutputtokens2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmoutputtokens2Inputs */ - -const en_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Output tokens per scaffold`) -}; - -const es_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tokens de salida por scaffold`) -}; - -const zh_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`每次 scaffold 输出 tokens`) -}; - -const ja_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`スキャフォールドごとの出力トークン`) -}; - -const ko_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`스캐폴드당 출력 토큰`) -}; - -const zh_hant1_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`每次 scaffold 輸出 tokens`) -}; - -const de_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ausgabetoken pro Gerüst`) -}; - -const fr_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Jetons de sortie par échafaudage`) -}; - -const uk_llmoutputtokens2 = /** @type {(inputs: Llmoutputtokens2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Output-токени на скафолд`) -}; - -/** -* | output | -* | --- | -* | "Output tokens per scaffold" | -* -* @param {Llmoutputtokens2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmoutputtokens2 = /** @type {((inputs?: Llmoutputtokens2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmoutputtokens2(inputs) - if (locale === "zh") return zh_llmoutputtokens2(inputs) - if (locale === "ja") return ja_llmoutputtokens2(inputs) - if (locale === "ko") return ko_llmoutputtokens2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmoutputtokens2(inputs) - if (locale === "de") return de_llmoutputtokens2(inputs) - if (locale === "fr") return fr_llmoutputtokens2(inputs) - if (locale === "uk") return uk_llmoutputtokens2(inputs) - return en_llmoutputtokens2(inputs) -}); -export { llmoutputtokens2 as "llmOutputTokens" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmpathclidetail3.js b/apps/web/src/paraglide/messages/llmpathclidetail3.js deleted file mode 100644 index 443f676df..000000000 --- a/apps/web/src/paraglide/messages/llmpathclidetail3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmpathclidetail3Inputs */ - -const en_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`agent composes the Better-Fullstack CLI command`) -}; - -const es_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`el agente compone el comando CLI de Better-Fullstack`) -}; - -const zh_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`代理组合 Better-Fullstack CLI 命令`) -}; - -const ja_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`エージェントは Better-Fullstack CLI コマンドを作成します`) -}; - -const ko_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`에이전트는 Better-Fullstack CLI 명령을 구성합니다.`) -}; - -const zh_hant1_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`代理程式組合 Better-Fullstack CLI 指令`) -}; - -const de_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Der Agent erstellt den Befehl Better-Fullstack CLI`) -}; - -const fr_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`l'agent compose la commande Better-Fullstack CLI`) -}; - -const uk_llmpathclidetail3 = /** @type {(inputs: Llmpathclidetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`агент складає команду Better Fullstack CLI`) -}; - -/** -* | output | -* | --- | -* | "agent composes the Better-Fullstack CLI command" | -* -* @param {Llmpathclidetail3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmpathclidetail3 = /** @type {((inputs?: Llmpathclidetail3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmpathclidetail3(inputs) - if (locale === "zh") return zh_llmpathclidetail3(inputs) - if (locale === "ja") return ja_llmpathclidetail3(inputs) - if (locale === "ko") return ko_llmpathclidetail3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmpathclidetail3(inputs) - if (locale === "de") return de_llmpathclidetail3(inputs) - if (locale === "fr") return fr_llmpathclidetail3(inputs) - if (locale === "uk") return uk_llmpathclidetail3(inputs) - return en_llmpathclidetail3(inputs) -}); -export { llmpathclidetail3 as "llmPathCliDetail" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmpathclishort3.js b/apps/web/src/paraglide/messages/llmpathclishort3.js deleted file mode 100644 index 49a2159a4..000000000 --- a/apps/web/src/paraglide/messages/llmpathclishort3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmpathclishort3Inputs */ - -const en_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`BF mention`) -}; - -const es_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Mención BF`) -}; - -const zh_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`BF 提及`) -}; - -const ja_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`BFの言及`) -}; - -const ko_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`BF 언급`) -}; - -const zh_hant1_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`BF 提及`) -}; - -const de_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`BF-Erwähnung`) -}; - -const fr_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Mention BF`) -}; - -const uk_llmpathclishort3 = /** @type {(inputs: Llmpathclishort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`BF mention`) -}; - -/** -* | output | -* | --- | -* | "BF mention" | -* -* @param {Llmpathclishort3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmpathclishort3 = /** @type {((inputs?: Llmpathclishort3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmpathclishort3(inputs) - if (locale === "zh") return zh_llmpathclishort3(inputs) - if (locale === "ja") return ja_llmpathclishort3(inputs) - if (locale === "ko") return ko_llmpathclishort3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmpathclishort3(inputs) - if (locale === "de") return de_llmpathclishort3(inputs) - if (locale === "fr") return fr_llmpathclishort3(inputs) - if (locale === "uk") return uk_llmpathclishort3(inputs) - return en_llmpathclishort3(inputs) -}); -export { llmpathclishort3 as "llmPathCliShort" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmpathmcpdetail3.js b/apps/web/src/paraglide/messages/llmpathmcpdetail3.js deleted file mode 100644 index 59c63c20e..000000000 --- a/apps/web/src/paraglide/messages/llmpathmcpdetail3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmpathmcpdetail3Inputs */ - -const en_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`scaffolds through our MCP tools`) -}; - -const es_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`crea el scaffold mediante nuestras herramientas MCP`) -}; - -const zh_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`通过我们的 MCP 工具生成 scaffold`) -}; - -const ja_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`MCP ツールを使用した足場`) -}; - -const ko_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`MCP 도구를 통해 스캐폴딩합니다.`) -}; - -const zh_hant1_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`透過我們的 MCP 工具產生 scaffold`) -}; - -const de_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`erstellt Gerüste über unsere MCP-Tools`) -}; - -const fr_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`échafaudages grâce à nos outils MCP`) -}; - -const uk_llmpathmcpdetail3 = /** @type {(inputs: Llmpathmcpdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`генерує через наші MCP-інструменти`) -}; - -/** -* | output | -* | --- | -* | "scaffolds through our MCP tools" | -* -* @param {Llmpathmcpdetail3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmpathmcpdetail3 = /** @type {((inputs?: Llmpathmcpdetail3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmpathmcpdetail3(inputs) - if (locale === "zh") return zh_llmpathmcpdetail3(inputs) - if (locale === "ja") return ja_llmpathmcpdetail3(inputs) - if (locale === "ko") return ko_llmpathmcpdetail3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmpathmcpdetail3(inputs) - if (locale === "de") return de_llmpathmcpdetail3(inputs) - if (locale === "fr") return fr_llmpathmcpdetail3(inputs) - if (locale === "uk") return uk_llmpathmcpdetail3(inputs) - return en_llmpathmcpdetail3(inputs) -}); -export { llmpathmcpdetail3 as "llmPathMcpDetail" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmpathpromptdetail3.js b/apps/web/src/paraglide/messages/llmpathpromptdetail3.js deleted file mode 100644 index 6316d9ff6..000000000 --- a/apps/web/src/paraglide/messages/llmpathpromptdetail3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmpathpromptdetail3Inputs */ - -const en_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`no Better-Fullstack - agent hand-writes every file`) -}; - -const es_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`sin Better-Fullstack: el agente escribe cada archivo a mano`) -}; - -const zh_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`不使用 Better-Fullstack:代理手写每个文件`) -}; - -const ja_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Better-Fullstack なし - エージェントがすべてのファイルを手書きします`) -}; - -const ko_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Better-Fullstack 없음 - 에이전트가 모든 파일을 직접 작성합니다.`) -}; - -const zh_hant1_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`不使用 Better-Fullstack:代理程式手寫每個文件`) -}; - -const de_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`kein Better-Fullstack – der Agent schreibt jede Datei von Hand`) -}; - -const fr_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`pas de Better-Fullstack - l'agent écrit manuellement chaque fichier`) -}; - -const uk_llmpathpromptdetail3 = /** @type {(inputs: Llmpathpromptdetail3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`без Better Fullstack - агент вручну пише кожен файл`) -}; - -/** -* | output | -* | --- | -* | "no Better-Fullstack - agent hand-writes every file" | -* -* @param {Llmpathpromptdetail3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmpathpromptdetail3 = /** @type {((inputs?: Llmpathpromptdetail3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmpathpromptdetail3(inputs) - if (locale === "zh") return zh_llmpathpromptdetail3(inputs) - if (locale === "ja") return ja_llmpathpromptdetail3(inputs) - if (locale === "ko") return ko_llmpathpromptdetail3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmpathpromptdetail3(inputs) - if (locale === "de") return de_llmpathpromptdetail3(inputs) - if (locale === "fr") return fr_llmpathpromptdetail3(inputs) - if (locale === "uk") return uk_llmpathpromptdetail3(inputs) - return en_llmpathpromptdetail3(inputs) -}); -export { llmpathpromptdetail3 as "llmPathPromptDetail" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmpathpromptshort3.js b/apps/web/src/paraglide/messages/llmpathpromptshort3.js deleted file mode 100644 index 3d3488502..000000000 --- a/apps/web/src/paraglide/messages/llmpathpromptshort3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmpathpromptshort3Inputs */ - -const en_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prompt`) -}; - -const es_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prompt`) -}; - -const zh_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prompt`) -}; - -const ja_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`プロンプト`) -}; - -const ko_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`프롬프트`) -}; - -const zh_hant1_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prompt`) -}; - -const de_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prompt`) -}; - -const fr_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Invite`) -}; - -const uk_llmpathpromptshort3 = /** @type {(inputs: Llmpathpromptshort3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prompt`) -}; - -/** -* | output | -* | --- | -* | "Prompt" | -* -* @param {Llmpathpromptshort3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmpathpromptshort3 = /** @type {((inputs?: Llmpathpromptshort3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmpathpromptshort3(inputs) - if (locale === "zh") return zh_llmpathpromptshort3(inputs) - if (locale === "ja") return ja_llmpathpromptshort3(inputs) - if (locale === "ko") return ko_llmpathpromptshort3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmpathpromptshort3(inputs) - if (locale === "de") return de_llmpathpromptshort3(inputs) - if (locale === "fr") return fr_llmpathpromptshort3(inputs) - if (locale === "uk") return uk_llmpathpromptshort3(inputs) - return en_llmpathpromptshort3(inputs) -}); -export { llmpathpromptshort3 as "llmPathPromptShort" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmreadblog2.js b/apps/web/src/paraglide/messages/llmreadblog2.js deleted file mode 100644 index 6766d388f..000000000 --- a/apps/web/src/paraglide/messages/llmreadblog2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmreadblog2Inputs */ - -const en_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Read the blog`) -}; - -const es_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Leer el blog`) -}; - -const zh_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`阅读博客`) -}; - -const ja_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ブログを読む`) -}; - -const ko_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`블로그 읽기`) -}; - -const zh_hant1_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`閱讀部落格`) -}; - -const de_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Lesen Sie den Blog`) -}; - -const fr_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Lire le blog`) -}; - -const uk_llmreadblog2 = /** @type {(inputs: Llmreadblog2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Читати блог`) -}; - -/** -* | output | -* | --- | -* | "Read the blog" | -* -* @param {Llmreadblog2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmreadblog2 = /** @type {((inputs?: Llmreadblog2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmreadblog2(inputs) - if (locale === "zh") return zh_llmreadblog2(inputs) - if (locale === "ja") return ja_llmreadblog2(inputs) - if (locale === "ko") return ko_llmreadblog2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmreadblog2(inputs) - if (locale === "de") return de_llmreadblog2(inputs) - if (locale === "fr") return fr_llmreadblog2(inputs) - if (locale === "uk") return uk_llmreadblog2(inputs) - return en_llmreadblog2(inputs) -}); -export { llmreadblog2 as "llmReadBlog" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmrunityourself3.js b/apps/web/src/paraglide/messages/llmrunityourself3.js deleted file mode 100644 index 246225531..000000000 --- a/apps/web/src/paraglide/messages/llmrunityourself3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmrunityourself3Inputs */ - -const en_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Run it yourself`) -}; - -const es_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ejecútalo tú mismo`) -}; - -const zh_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`自己运行`) -}; - -const ja_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`自分で実行してください`) -}; - -const ko_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`직접 실행해 보세요`) -}; - -const zh_hant1_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`自己運行`) -}; - -const de_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Führen Sie es selbst aus`) -}; - -const fr_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Exécutez-le vous-même`) -}; - -const uk_llmrunityourself3 = /** @type {(inputs: Llmrunityourself3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Запустіть самостійно`) -}; - -/** -* | output | -* | --- | -* | "Run it yourself" | -* -* @param {Llmrunityourself3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmrunityourself3 = /** @type {((inputs?: Llmrunityourself3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmrunityourself3(inputs) - if (locale === "zh") return zh_llmrunityourself3(inputs) - if (locale === "ja") return ja_llmrunityourself3(inputs) - if (locale === "ko") return ko_llmrunityourself3(inputs) - if (locale === "zh-Hant") return zh_hant1_llmrunityourself3(inputs) - if (locale === "de") return de_llmrunityourself3(inputs) - if (locale === "fr") return fr_llmrunityourself3(inputs) - if (locale === "uk") return uk_llmrunityourself3(inputs) - return en_llmrunityourself3(inputs) -}); -export { llmrunityourself3 as "llmRunItYourself" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmscatteraria2.js b/apps/web/src/paraglide/messages/llmscatteraria2.js deleted file mode 100644 index 5ecb13b32..000000000 --- a/apps/web/src/paraglide/messages/llmscatteraria2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmscatteraria2Inputs */ - -const en_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark scatter chart: each point is one model and creation path`) -}; - -const es_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Gráfico de dispersión del benchmark: cada punto es un modelo y una ruta de creación`) -}; - -const zh_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark 散点图:每个点代表一个模型和一种创建路径`) -}; - -const ja_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ベンチマーク散布図: 各ポイントは 1 つのモデルと作成パスです`) -}; - -const ko_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`벤치마크 분산형 차트: 각 지점은 하나의 모델이자 생성 경로입니다.`) -}; - -const zh_hant1_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark 散佈圖:每個點代表一個模型和一種建立路徑`) -}; - -const de_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark-Streudiagramm: Jeder Punkt ist ein Modell und ein Erstellungspfad`) -}; - -const fr_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Diagramme de dispersion de référence : chaque point correspond à un modèle et à un chemin de création`) -}; - -const uk_llmscatteraria2 = /** @type {(inputs: Llmscatteraria2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Scatter chart бенчмарку: кожна точка - одна модель і шлях створення`) -}; - -/** -* | output | -* | --- | -* | "Benchmark scatter chart: each point is one model and creation path" | -* -* @param {Llmscatteraria2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmscatteraria2 = /** @type {((inputs?: Llmscatteraria2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmscatteraria2(inputs) - if (locale === "zh") return zh_llmscatteraria2(inputs) - if (locale === "ja") return ja_llmscatteraria2(inputs) - if (locale === "ko") return ko_llmscatteraria2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmscatteraria2(inputs) - if (locale === "de") return de_llmscatteraria2(inputs) - if (locale === "fr") return fr_llmscatteraria2(inputs) - if (locale === "uk") return uk_llmscatteraria2(inputs) - return en_llmscatteraria2(inputs) -}); -export { llmscatteraria2 as "llmScatterAria" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmscatterunmetered2.js b/apps/web/src/paraglide/messages/llmscatterunmetered2.js deleted file mode 100644 index 1e302aef8..000000000 --- a/apps/web/src/paraglide/messages/llmscatterunmetered2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{ models: NonNullable }} Llmscatterunmetered2Inputs */ - -const en_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`Not metered on this axis (excluded from the plot): ${i?.models}`) -}; - -const es_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`Sin medición en este eje (excluido del gráfico): ${i?.models}`) -}; - -const zh_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`此坐标轴未计量(未绘制在图中):${i?.models}`) -}; - -const ja_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`この軸では計測されていません(プロットから除外):${i?.models}`) -}; - -const ko_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`이 축에서는 측정되지 않음(차트에서 제외됨): ${i?.models}`) -}; - -const zh_hant1_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`此座標軸未計量(未繪製在圖中):${i?.models}`) -}; - -const de_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`Auf dieser Achse nicht gemessen (nicht im Diagramm): ${i?.models}`) -}; - -const fr_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`Non mesuré sur cet axe (exclu du graphique) : ${i?.models}`) -}; - -const uk_llmscatterunmetered2 = /** @type {(inputs: Llmscatterunmetered2Inputs) => LocalizedString} */ (i) => { - return /** @type {LocalizedString} */ (`Не вимірюється на цій осі (виключено з графіка): ${i?.models}`) -}; - -/** -* | output | -* | --- | -* | "Not metered on this axis (excluded from the plot): {models}" | -* -* @param {Llmscatterunmetered2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmscatterunmetered2 = /** @type {((inputs: Llmscatterunmetered2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmscatterunmetered2(inputs) - if (locale === "zh") return zh_llmscatterunmetered2(inputs) - if (locale === "ja") return ja_llmscatterunmetered2(inputs) - if (locale === "ko") return ko_llmscatterunmetered2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmscatterunmetered2(inputs) - if (locale === "de") return de_llmscatterunmetered2(inputs) - if (locale === "fr") return fr_llmscatterunmetered2(inputs) - if (locale === "uk") return uk_llmscatterunmetered2(inputs) - return en_llmscatterunmetered2(inputs) -}); -export { llmscatterunmetered2 as "llmScatterUnmetered" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmspeed1.js b/apps/web/src/paraglide/messages/llmspeed1.js deleted file mode 100644 index 479ee1312..000000000 --- a/apps/web/src/paraglide/messages/llmspeed1.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmspeed1Inputs */ - -const en_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Speed`) -}; - -const es_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Velocidad`) -}; - -const zh_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`速度`) -}; - -const ja_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`スピード`) -}; - -const ko_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`속도`) -}; - -const zh_hant1_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`速度`) -}; - -const de_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Geschwindigkeit`) -}; - -const fr_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Vitesse`) -}; - -const uk_llmspeed1 = /** @type {(inputs: Llmspeed1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Швидкість`) -}; - -/** -* | output | -* | --- | -* | "Speed" | -* -* @param {Llmspeed1Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmspeed1 = /** @type {((inputs?: Llmspeed1Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmspeed1(inputs) - if (locale === "zh") return zh_llmspeed1(inputs) - if (locale === "ja") return ja_llmspeed1(inputs) - if (locale === "ko") return ko_llmspeed1(inputs) - if (locale === "zh-Hant") return zh_hant1_llmspeed1(inputs) - if (locale === "de") return de_llmspeed1(inputs) - if (locale === "fr") return fr_llmspeed1(inputs) - if (locale === "uk") return uk_llmspeed1(inputs) - return en_llmspeed1(inputs) -}); -export { llmspeed1 as "llmSpeed" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmtokens1.js b/apps/web/src/paraglide/messages/llmtokens1.js deleted file mode 100644 index 0c3c2ec0c..000000000 --- a/apps/web/src/paraglide/messages/llmtokens1.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmtokens1Inputs */ - -const en_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tokens`) -}; - -const es_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tokens`) -}; - -const zh_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tokens`) -}; - -const ja_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`トークン`) -}; - -const ko_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`토큰`) -}; - -const zh_hant1_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tokens`) -}; - -const de_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Token`) -}; - -const fr_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Jetons`) -}; - -const uk_llmtokens1 = /** @type {(inputs: Llmtokens1Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Токени`) -}; - -/** -* | output | -* | --- | -* | "Tokens" | -* -* @param {Llmtokens1Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmtokens1 = /** @type {((inputs?: Llmtokens1Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmtokens1(inputs) - if (locale === "zh") return zh_llmtokens1(inputs) - if (locale === "ja") return ja_llmtokens1(inputs) - if (locale === "ko") return ko_llmtokens1(inputs) - if (locale === "zh-Hant") return zh_hant1_llmtokens1(inputs) - if (locale === "de") return de_llmtokens1(inputs) - if (locale === "fr") return fr_llmtokens1(inputs) - if (locale === "uk") return uk_llmtokens1(inputs) - return en_llmtokens1(inputs) -}); -export { llmtokens1 as "llmTokens" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/llmtrymcp2.js b/apps/web/src/paraglide/messages/llmtrymcp2.js deleted file mode 100644 index 5873cd129..000000000 --- a/apps/web/src/paraglide/messages/llmtrymcp2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Llmtrymcp2Inputs */ - -const en_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Try out MCP`) -}; - -const es_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Probar MCP`) -}; - -const zh_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`试用 MCP`) -}; - -const ja_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`MCP を試してみる`) -}; - -const ko_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`MCP를 사용해 보세요.`) -}; - -const zh_hant1_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`試試 MCP`) -}; - -const de_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Probieren Sie MCP aus`) -}; - -const fr_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Essayez MCP`) -}; - -const uk_llmtrymcp2 = /** @type {(inputs: Llmtrymcp2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Спробувати MCP`) -}; - -/** -* | output | -* | --- | -* | "Try out MCP" | -* -* @param {Llmtrymcp2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const llmtrymcp2 = /** @type {((inputs?: Llmtrymcp2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_llmtrymcp2(inputs) - if (locale === "zh") return zh_llmtrymcp2(inputs) - if (locale === "ja") return ja_llmtrymcp2(inputs) - if (locale === "ko") return ko_llmtrymcp2(inputs) - if (locale === "zh-Hant") return zh_hant1_llmtrymcp2(inputs) - if (locale === "de") return de_llmtrymcp2(inputs) - if (locale === "fr") return fr_llmtrymcp2(inputs) - if (locale === "uk") return uk_llmtrymcp2(inputs) - return en_llmtrymcp2(inputs) -}); -export { llmtrymcp2 as "llmTryMcp" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/mcpfinaldescription2.js b/apps/web/src/paraglide/messages/mcpfinaldescription2.js index ae235d23a..0a863e3d5 100644 --- a/apps/web/src/paraglide/messages/mcpfinaldescription2.js +++ b/apps/web/src/paraglide/messages/mcpfinaldescription2.js @@ -6,45 +6,45 @@ import { getLocale, experimentalStaticLocale } from '../runtime.js'; /** @typedef {{}} Mcpfinaldescription2Inputs */ const en_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`In ScaffBench, MCP-guided project creation is faster and more reliable than asking an agent to hand-write a project from scratch.`) + return /** @type {LocalizedString} */ (`Fixproof grades coding agents on sealed, real issues from private and public codebases, verified by hidden tests.`) }; const es_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`En ScaffBench, crear proyectos guiados por MCP es más rápido y fiable que pedirle a un agente que escriba todo desde cero.`) + return /** @type {LocalizedString} */ (`Fixproof evalúa agentes de programación con errores reales y sellados de bases de código privadas y públicas, verificados por pruebas ocultas.`) }; const zh_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`在 ScaffBench 中,由 MCP 引导的项目创建比让代理从零手写项目更快也更可靠。`) + return /** @type {LocalizedString} */ (`Fixproof 用来自私有和公开代码库的封闭真实问题评测编程代理,并由隐藏测试验证。`) }; const ja_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench では、MCP のガイド付きプロジェクト作成は、エージェントにプロジェクトを最初から手書きで作成させるよりも速く、信頼性が高くなります。`) + return /** @type {LocalizedString} */ (`Fixproof は、非公開および公開コードベースから集めた封印済みの実際の不具合でコーディングエージェントを採点し、非公開テストで検証します。`) }; const ko_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench에서는 MCP 안내에 따라 프로젝트를 생성하는 것이 에이전트에게 프로젝트를 처음부터 직접 작성하도록 요청하는 것보다 더 빠르고 안정적입니다.`) + return /** @type {LocalizedString} */ (`Fixproof는 비공개 및 공개 코드베이스에서 가져온 봉인된 실제 이슈로 코딩 에이전트를 채점하고, 비공개 테스트로 검증합니다.`) }; const zh_hant1_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`在 ScaffBench 中,由 MCP 引導的專案建立比讓代理從零手寫專案更快也更可靠。`) + return /** @type {LocalizedString} */ (`Fixproof 用來自私有和公開程式碼庫的封閉真實問題評測程式代理程式,並由隱藏測試驗證。`) }; const de_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`In ScaffBench ist die MCP-geführte Projekterstellung schneller und zuverlässiger, als einen Agenten zu bitten, ein Projekt von Grund auf handschriftlich zu schreiben.`) + return /** @type {LocalizedString} */ (`Fixproof bewertet Coding-Agenten an versiegelten, echten Fehlern aus privaten und öffentlichen Codebasen, verifiziert durch verborgene Tests.`) }; const fr_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Dans ScaffBench, la création de projet guidée par MCP est plus rapide et plus fiable que de demander à un agent d'écrire manuellement un projet à partir de zéro.`) + return /** @type {LocalizedString} */ (`Fixproof évalue les agents de codage sur des bugs réels et scellés issus de bases de code privées et publiques, vérifiés par des tests cachés.`) }; const uk_mcpfinaldescription2 = /** @type {(inputs: Mcpfinaldescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`У ScaffBench створення проєкту через MCP швидше й надійніше, ніж просити агента писати проєкт вручну з нуля.`) + return /** @type {LocalizedString} */ (`Fixproof оцінює агентів для коду на закритих реальних помилках із приватних і публічних кодових баз, перевірених прихованими тестами.`) }; /** * | output | * | --- | -* | "In ScaffBench, MCP-guided project creation is faster and more reliable than asking an agent to hand-write a project from scratch." | +* | "Fixproof grades coding agents on sealed, real issues from private and public codebases, verified by hidden tests." | * * @param {Mcpfinaldescription2Inputs} inputs * @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options diff --git a/apps/web/src/paraglide/messages/mcpfinaleyebrow2.js b/apps/web/src/paraglide/messages/mcpfinaleyebrow2.js index 3de8c79ac..a119fc635 100644 --- a/apps/web/src/paraglide/messages/mcpfinaleyebrow2.js +++ b/apps/web/src/paraglide/messages/mcpfinaleyebrow2.js @@ -6,45 +6,45 @@ import { getLocale, experimentalStaticLocale } from '../runtime.js'; /** @typedef {{}} Mcpfinaleyebrow2Inputs */ const en_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`benchmark-backed`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const es_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`respaldado por benchmark`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const zh_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`由 benchmark 支撑`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const ja_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ベンチマークに裏付けられた`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const ko_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`벤치마크 지원`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const zh_hant1_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`由 benchmark 支撐`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const de_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Benchmark-gestützt`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const fr_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`appuyé par un benchmark`) + return /** @type {LocalizedString} */ (`Fixproof`) }; const uk_mcpfinaleyebrow2 = /** @type {(inputs: Mcpfinaleyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`підтверджено бенчмарком`) + return /** @type {LocalizedString} */ (`Fixproof`) }; /** * | output | * | --- | -* | "benchmark-backed" | +* | "Fixproof" | * * @param {Mcpfinaleyebrow2Inputs} inputs * @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options diff --git a/apps/web/src/paraglide/messages/mcpfinaltitle2.js b/apps/web/src/paraglide/messages/mcpfinaltitle2.js index 7fa2d34ca..e9f97c6ef 100644 --- a/apps/web/src/paraglide/messages/mcpfinaltitle2.js +++ b/apps/web/src/paraglide/messages/mcpfinaltitle2.js @@ -6,45 +6,45 @@ import { getLocale, experimentalStaticLocale } from '../runtime.js'; /** @typedef {{}} Mcpfinaltitle2Inputs */ const en_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2.6× faster than`) + return /** @type {LocalizedString} */ (`See how agents score on`) }; const es_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2.6× más rápido que`) + return /** @type {LocalizedString} */ (`Mira cómo puntúan los agentes con`) }; const zh_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`比`) + return /** @type {LocalizedString} */ (`在真实问题上`) }; const ja_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`プロンプトのみより`) + return /** @type {LocalizedString} */ (`エージェントの成績が分かるのは`) }; const ko_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2.6배 더 빠르다`) + return /** @type {LocalizedString} */ (`에이전트의 성적을 보여 주는 기준은`) }; const zh_hant1_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`比`) + return /** @type {LocalizedString} */ (`在真實問題上`) }; const de_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2,6× schneller als`) + return /** @type {LocalizedString} */ (`Sehen Sie, wie Agenten abschneiden bei`) }; const fr_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2,6 fois plus rapide que`) + return /** @type {LocalizedString} */ (`Voyez comment les agents s'en sortent sur`) }; const uk_mcpfinaltitle2 = /** @type {(inputs: Mcpfinaltitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2,6× швидше, ніж`) + return /** @type {LocalizedString} */ (`Подивіться, як агенти показують себе на`) }; /** * | output | * | --- | -* | "2.6× faster than" | +* | "See how agents score on" | * * @param {Mcpfinaltitle2Inputs} inputs * @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options diff --git a/apps/web/src/paraglide/messages/mcpfinaltitleemphasis3.js b/apps/web/src/paraglide/messages/mcpfinaltitleemphasis3.js index 85462facb..78503f198 100644 --- a/apps/web/src/paraglide/messages/mcpfinaltitleemphasis3.js +++ b/apps/web/src/paraglide/messages/mcpfinaltitleemphasis3.js @@ -6,45 +6,45 @@ import { getLocale, experimentalStaticLocale } from '../runtime.js'; /** @typedef {{}} Mcpfinaltitleemphasis3Inputs */ const en_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt-only.`) + return /** @type {LocalizedString} */ (`real issues.`) }; const es_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`solo prompt.`) + return /** @type {LocalizedString} */ (`errores reales.`) }; const zh_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`纯 prompt 快 2.6×。`) + return /** @type {LocalizedString} */ (`查看代理的得分。`) }; const ja_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2.6倍高速。`) + return /** @type {LocalizedString} */ (`実際の不具合。`) }; const ko_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`프롬프트 전용.`) + return /** @type {LocalizedString} */ (`실제 이슈.`) }; const zh_hant1_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`純 prompt 快 2.6×。`) + return /** @type {LocalizedString} */ (`查看代理程式的得分。`) }; const de_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Nur per Prompt.`) + return /** @type {LocalizedString} */ (`echten Fehlern.`) }; const fr_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`invite uniquement.`) + return /** @type {LocalizedString} */ (`des bugs réels.`) }; const uk_mcpfinaltitleemphasis3 = /** @type {(inputs: Mcpfinaltitleemphasis3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt-only.`) + return /** @type {LocalizedString} */ (`реальних помилках.`) }; /** * | output | * | --- | -* | "prompt-only." | +* | "real issues." | * * @param {Mcpfinaltitleemphasis3Inputs} inputs * @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options diff --git a/apps/web/src/paraglide/messages/mcpstatfasterpromptonly4.js b/apps/web/src/paraglide/messages/mcpstatfasterpromptonly4.js deleted file mode 100644 index ce0f0f18f..000000000 --- a/apps/web/src/paraglide/messages/mcpstatfasterpromptonly4.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Mcpstatfasterpromptonly4Inputs */ - -const en_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`faster than prompt-only`) -}; - -const es_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`más rápido que solo prompt`) -}; - -const zh_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`比纯 prompt 更快`) -}; - -const ja_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`プロンプトのみよりも高速`) -}; - -const ko_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`프롬프트 전용보다 빠릅니다.`) -}; - -const zh_hant1_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`比純 prompt 更快`) -}; - -const de_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`schneller als nur per Prompt`) -}; - -const fr_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`plus rapide que l'invite uniquement`) -}; - -const uk_mcpstatfasterpromptonly4 = /** @type {(inputs: Mcpstatfasterpromptonly4Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`швидше за prompt-only`) -}; - -/** -* | output | -* | --- | -* | "faster than prompt-only" | -* -* @param {Mcpstatfasterpromptonly4Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const mcpstatfasterpromptonly4 = /** @type {((inputs?: Mcpstatfasterpromptonly4Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_mcpstatfasterpromptonly4(inputs) - if (locale === "zh") return zh_mcpstatfasterpromptonly4(inputs) - if (locale === "ja") return ja_mcpstatfasterpromptonly4(inputs) - if (locale === "ko") return ko_mcpstatfasterpromptonly4(inputs) - if (locale === "zh-Hant") return zh_hant1_mcpstatfasterpromptonly4(inputs) - if (locale === "de") return de_mcpstatfasterpromptonly4(inputs) - if (locale === "fr") return fr_mcpstatfasterpromptonly4(inputs) - if (locale === "uk") return uk_mcpstatfasterpromptonly4(inputs) - return en_mcpstatfasterpromptonly4(inputs) -}); -export { mcpstatfasterpromptonly4 as "mcpStatFasterPromptOnly" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runagentsdesc2.js b/apps/web/src/paraglide/messages/runagentsdesc2.js deleted file mode 100644 index d98ffddab..000000000 --- a/apps/web/src/paraglide/messages/runagentsdesc2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runagentsdesc2Inputs */ - -const en_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`The provider is inferred from the model id, so one flag picks both the model and the CLI that drives it.`) -}; - -const es_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`El proveedor se infiere del id del modelo, así que un solo flag selecciona tanto el modelo como la CLI que lo controla.`) -}; - -const zh_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`提供商由模型 ID 推断得出,因此一个 flag 会同时选定模型和驱动它的 CLI。`) -}; - -const ja_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`プロバイダーはモデルIDから推測されるため、1つのフラグでモデルと、それを駆動するCLIの両方を選択できます。`) -}; - -const ko_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`제공자는 모델 ID에서 추론되므로 하나의 플래그로 모델과 해당 모델을 구동하는 CLI를 모두 선택할 수 있습니다.`) -}; - -const zh_hant1_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`提供者由模型 ID 推斷得出,因此一個旗標就能同時選定模型與驅動它的 CLI。`) -}; - -const de_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Der Provider wird aus der Modell-ID abgeleitet, sodass ein Flag sowohl das Modell als auch die zugehörige CLI auswählt.`) -}; - -const fr_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Le fournisseur est déduit de l'identifiant du modèle ; un seul indicateur sélectionne donc à la fois le modèle et l'interface de ligne de commande qui le pilote.`) -}; - -const uk_runagentsdesc2 = /** @type {(inputs: Runagentsdesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Постачальник визначається за ідентифікатором моделі, тому один прапор вибирає як модель, так і CLI, який нею керує.`) -}; - -/** -* | output | -* | --- | -* | "The provider is inferred from the model id, so one flag picks both the model and the CLI that drives it." | -* -* @param {Runagentsdesc2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runagentsdesc2 = /** @type {((inputs?: Runagentsdesc2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runagentsdesc2(inputs) - if (locale === "zh") return zh_runagentsdesc2(inputs) - if (locale === "ja") return ja_runagentsdesc2(inputs) - if (locale === "ko") return ko_runagentsdesc2(inputs) - if (locale === "zh-Hant") return zh_hant1_runagentsdesc2(inputs) - if (locale === "de") return de_runagentsdesc2(inputs) - if (locale === "fr") return fr_runagentsdesc2(inputs) - if (locale === "uk") return uk_runagentsdesc2(inputs) - return en_runagentsdesc2(inputs) -}); -export { runagentsdesc2 as "runAgentsDesc" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runagentseyebrow2.js b/apps/web/src/paraglide/messages/runagentseyebrow2.js deleted file mode 100644 index 664accb1c..000000000 --- a/apps/web/src/paraglide/messages/runagentseyebrow2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runagentseyebrow2Inputs */ - -const en_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agents & models`) -}; - -const es_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agentes y modelos`) -}; - -const zh_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`代理与模型`) -}; - -const ja_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`エージェントとモデル`) -}; - -const ko_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`에이전트 및 모델`) -}; - -const zh_hant1_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`代理與模型`) -}; - -const de_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agenten & Models`) -}; - -const fr_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agents et modèles`) -}; - -const uk_runagentseyebrow2 = /** @type {(inputs: Runagentseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Агенти та моделі`) -}; - -/** -* | output | -* | --- | -* | "Agents & models" | -* -* @param {Runagentseyebrow2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runagentseyebrow2 = /** @type {((inputs?: Runagentseyebrow2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runagentseyebrow2(inputs) - if (locale === "zh") return zh_runagentseyebrow2(inputs) - if (locale === "ja") return ja_runagentseyebrow2(inputs) - if (locale === "ko") return ko_runagentseyebrow2(inputs) - if (locale === "zh-Hant") return zh_hant1_runagentseyebrow2(inputs) - if (locale === "de") return de_runagentseyebrow2(inputs) - if (locale === "fr") return fr_runagentseyebrow2(inputs) - if (locale === "uk") return uk_runagentseyebrow2(inputs) - return en_runagentseyebrow2(inputs) -}); -export { runagentseyebrow2 as "runAgentsEyebrow" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runagentstitle2.js b/apps/web/src/paraglide/messages/runagentstitle2.js deleted file mode 100644 index ff18c57b3..000000000 --- a/apps/web/src/paraglide/messages/runagentstitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runagentstitle2Inputs */ - -const en_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Bring any agent`) -}; - -const es_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Trae cualquier agente`) -}; - -const zh_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`接入任意代理`) -}; - -const ja_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`どんなエージェントでも`) -}; - -const ko_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`어떤 에이전트든 데려오세요`) -}; - -const zh_hant1_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`帶上任何代理人`) -}; - -const de_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Bringen Sie jeden beliebigen Agenten mit.`) -}; - -const fr_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Amenez n'importe quel agent`) -}; - -const uk_runagentstitle2 = /** @type {(inputs: Runagentstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Підключіть будь-якого агента`) -}; - -/** -* | output | -* | --- | -* | "Bring any agent" | -* -* @param {Runagentstitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runagentstitle2 = /** @type {((inputs?: Runagentstitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runagentstitle2(inputs) - if (locale === "zh") return zh_runagentstitle2(inputs) - if (locale === "ja") return ja_runagentstitle2(inputs) - if (locale === "ko") return ko_runagentstitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_runagentstitle2(inputs) - if (locale === "de") return de_runagentstitle2(inputs) - if (locale === "fr") return fr_runagentstitle2(inputs) - if (locale === "uk") return uk_runagentstitle2(inputs) - return en_runagentstitle2(inputs) -}); -export { runagentstitle2 as "runAgentsTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runauthapidesc3.js b/apps/web/src/paraglide/messages/runauthapidesc3.js deleted file mode 100644 index 59282a6f5..000000000 --- a/apps/web/src/paraglide/messages/runauthapidesc3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runauthapidesc3Inputs */ - -const en_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prefer an API key? Export the provider key and the same agent CLI bills against it - no subscription needed. We publish subscription-driven runs; API runs are untested but supported.`) -}; - -const es_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`¿Prefieres una clave API? Exporta la clave del proveedor y la misma CLI del agente facturará con ella; no se requiere suscripción. Publicamos ejecuciones con suscripción; las ejecuciones con API no están probadas, pero cuentan con soporte.`) -}; - -const zh_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`更倾向于使用 API 密钥?导出提供商密钥,即可使用同一代理 CLI 进行计费--无需订阅。我们发布基于订阅的运行结果;API 方式的运行未经测试,但同样受支持。`) -}; - -const ja_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`APIキーをご希望ですか?プロバイダーキーをエクスポートすれば、同じエージェントCLIがそのキーに基づいて課金します。サブスクリプションは不要です。弊社ではサブスクリプションベースの実行を公開していますが、API実行はテストされていませんがサポート対象です。`) -}; - -const ko_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`API 키를 선호하시나요? 공급자 키를 내보내면 동일한 에이전트 CLI에서 해당 키를 기준으로 요금이 청구됩니다. 구독이 필요하지 않습니다. 구독 기반 실행은 게시되지만 API 실행은 테스트되지 않았지만 지원됩니다.`) -}; - -const zh_hant1_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`更傾向於使用 API 金鑰?匯出提供者金鑰,即可使用相同代理 CLI 進行計費-無需訂閱。我們提供訂閱驅動的運行服務;API 運行服務未經測試,但我們提供支援。`) -}; - -const de_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Bevorzugen Sie einen API-Schlüssel? Exportieren Sie den Anbieterschlüssel, und die gleiche Agenten-CLI wird darüber abgerechnet – ein Abonnement ist nicht erforderlich. Wir veröffentlichen abonnementbasierte Ausführungen; API-Ausführungen sind ungetestet, werden aber unterstützt.`) -}; - -const fr_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Vous préférez une clé API ? Exportez la clé du fournisseur : la même interface de ligne de commande de l’agent facturera automatiquement l’exécution avec cette clé, sans abonnement. Nous publions les exécutions sur abonnement ; les exécutions via API ne sont pas testées, mais sont prises en charge.`) -}; - -const uk_runauthapidesc3 = /** @type {(inputs: Runauthapidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Бажаєте ключ API? Експортуйте ключ постачальника, і той самий агент CLI виставляє рахунки за нього - підписка не потрібна. Ми публікуємо прогони за підпискою; прогони через API не перевірені, але підтримуються.`) -}; - -/** -* | output | -* | --- | -* | "Prefer an API key? Export the provider key and the same agent CLI bills against it - no subscription needed. We publish subscription-driven runs; API runs ar..." | -* -* @param {Runauthapidesc3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runauthapidesc3 = /** @type {((inputs?: Runauthapidesc3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runauthapidesc3(inputs) - if (locale === "zh") return zh_runauthapidesc3(inputs) - if (locale === "ja") return ja_runauthapidesc3(inputs) - if (locale === "ko") return ko_runauthapidesc3(inputs) - if (locale === "zh-Hant") return zh_hant1_runauthapidesc3(inputs) - if (locale === "de") return de_runauthapidesc3(inputs) - if (locale === "fr") return fr_runauthapidesc3(inputs) - if (locale === "uk") return uk_runauthapidesc3(inputs) - return en_runauthapidesc3(inputs) -}); -export { runauthapidesc3 as "runAuthApiDesc" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runauthapitab3.js b/apps/web/src/paraglide/messages/runauthapitab3.js deleted file mode 100644 index 68ccfbe5c..000000000 --- a/apps/web/src/paraglide/messages/runauthapitab3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runauthapitab3Inputs */ - -const en_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`API key`) -}; - -const es_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`clave API`) -}; - -const zh_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`API 密钥`) -}; - -const ja_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`APIキー`) -}; - -const ko_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`API 키`) -}; - -const zh_hant1_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`API金鑰`) -}; - -const de_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`API-Schlüssel`) -}; - -const fr_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Clé API`) -}; - -const uk_runauthapitab3 = /** @type {(inputs: Runauthapitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ключ API`) -}; - -/** -* | output | -* | --- | -* | "API key" | -* -* @param {Runauthapitab3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runauthapitab3 = /** @type {((inputs?: Runauthapitab3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runauthapitab3(inputs) - if (locale === "zh") return zh_runauthapitab3(inputs) - if (locale === "ja") return ja_runauthapitab3(inputs) - if (locale === "ko") return ko_runauthapitab3(inputs) - if (locale === "zh-Hant") return zh_hant1_runauthapitab3(inputs) - if (locale === "de") return de_runauthapitab3(inputs) - if (locale === "fr") return fr_runauthapitab3(inputs) - if (locale === "uk") return uk_runauthapitab3(inputs) - return en_runauthapitab3(inputs) -}); -export { runauthapitab3 as "runAuthApiTab" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runauthclidesc3.js b/apps/web/src/paraglide/messages/runauthclidesc3.js deleted file mode 100644 index 9bbf5b389..000000000 --- a/apps/web/src/paraglide/messages/runauthclidesc3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runauthclidesc3Inputs */ - -const en_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Use an agent CLI you're already signed into (subscription / OAuth). Log in once, then the harness drives it - no keys in your environment.`) -}; - -const es_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Usa una CLI de agente en la que ya tengas sesión iniciada (suscripción/OAuth). Inicia sesión una vez y el harness se encarga del resto; sin claves en tu entorno.`) -}; - -const zh_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`使用你已登录的代理 CLI(订阅/OAuth)。只需登录一次,然后该框架即可驱动它--你的环境中无需任何密钥。`) -}; - -const ja_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`既にサインイン済みのエージェントCLI(サブスクリプション/OAuth)を使用してください。一度ログインすれば、あとはハーネスが自動的に操作します。環境内にキーは不要です。`) -}; - -const ko_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`이미 로그인한 에이전트 CLI(구독/OAuth)를 사용하세요. 한 번만 로그인하면 그 후에는 하네스가 자동으로 제어합니다. 환경에 키가 필요하지 않습니다.`) -}; - -const zh_hant1_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`使用你已登入的代理程式 CLI(訂閱/OAuth)。只需登入一次,然後該框架即可驅動它--你的環境中無需任何金鑰。`) -}; - -const de_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Verwenden Sie eine Agenten-CLI, bei der Sie bereits angemeldet sind (Abonnement/OAuth). Melden Sie sich einmal an, danach übernimmt das Framework die Steuerung – es sind keine Schlüssel in Ihrer Umgebung erforderlich.`) -}; - -const fr_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Utilisez l'interface de ligne de commande d'un agent auquel vous êtes déjà connecté (abonnement/OAuth). Connectez-vous une seule fois, puis le système prend le relais ; aucune clé n'est requise dans votre environnement.`) -}; - -const uk_runauthclidesc3 = /** @type {(inputs: Runauthclidesc3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Використовуйте agent CLI, у який ви вже ввійшли (підписка / OAuth). Увійдіть один раз, а далі harness керує запуском - без ключів у середовищі.`) -}; - -/** -* | output | -* | --- | -* | "Use an agent CLI you're already signed into (subscription / OAuth). Log in once, then the harness drives it - no keys in your environment." | -* -* @param {Runauthclidesc3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runauthclidesc3 = /** @type {((inputs?: Runauthclidesc3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runauthclidesc3(inputs) - if (locale === "zh") return zh_runauthclidesc3(inputs) - if (locale === "ja") return ja_runauthclidesc3(inputs) - if (locale === "ko") return ko_runauthclidesc3(inputs) - if (locale === "zh-Hant") return zh_hant1_runauthclidesc3(inputs) - if (locale === "de") return de_runauthclidesc3(inputs) - if (locale === "fr") return fr_runauthclidesc3(inputs) - if (locale === "uk") return uk_runauthclidesc3(inputs) - return en_runauthclidesc3(inputs) -}); -export { runauthclidesc3 as "runAuthCliDesc" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runauthclitab3.js b/apps/web/src/paraglide/messages/runauthclitab3.js deleted file mode 100644 index 8f6ff3668..000000000 --- a/apps/web/src/paraglide/messages/runauthclitab3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runauthclitab3Inputs */ - -const en_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Logged-in CLI`) -}; - -const es_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`CLI con sesión iniciada`) -}; - -const zh_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`已登录 CLI`) -}; - -const ja_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ログイン済みCLI`) -}; - -const ko_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`로그인된 CLI`) -}; - -const zh_hant1_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`已登入 CLI`) -}; - -const de_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Angemeldete CLI`) -}; - -const fr_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Interface de ligne de commande (CLI) connectée`) -}; - -const uk_runauthclitab3 = /** @type {(inputs: Runauthclitab3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Вхід через CLI`) -}; - -/** -* | output | -* | --- | -* | "Logged-in CLI" | -* -* @param {Runauthclitab3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runauthclitab3 = /** @type {((inputs?: Runauthclitab3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runauthclitab3(inputs) - if (locale === "zh") return zh_runauthclitab3(inputs) - if (locale === "ja") return ja_runauthclitab3(inputs) - if (locale === "ko") return ko_runauthclitab3(inputs) - if (locale === "zh-Hant") return zh_hant1_runauthclitab3(inputs) - if (locale === "de") return de_runauthclitab3(inputs) - if (locale === "fr") return fr_runauthclitab3(inputs) - if (locale === "uk") return uk_runauthclitab3(inputs) - return en_runauthclitab3(inputs) -}); -export { runauthclitab3 as "runAuthCliTab" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runcolagent2.js b/apps/web/src/paraglide/messages/runcolagent2.js deleted file mode 100644 index ecb0c068d..000000000 --- a/apps/web/src/paraglide/messages/runcolagent2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runcolagent2Inputs */ - -const en_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agent`) -}; - -const es_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agente`) -}; - -const zh_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`代理`) -}; - -const ja_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`エージェント`) -}; - -const ko_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`에이전트`) -}; - -const zh_hant1_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`代理人`) -}; - -const de_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agent`) -}; - -const fr_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Agent`) -}; - -const uk_runcolagent2 = /** @type {(inputs: Runcolagent2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Агент`) -}; - -/** -* | output | -* | --- | -* | "Agent" | -* -* @param {Runcolagent2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runcolagent2 = /** @type {((inputs?: Runcolagent2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runcolagent2(inputs) - if (locale === "zh") return zh_runcolagent2(inputs) - if (locale === "ja") return ja_runcolagent2(inputs) - if (locale === "ko") return ko_runcolagent2(inputs) - if (locale === "zh-Hant") return zh_hant1_runcolagent2(inputs) - if (locale === "de") return de_runcolagent2(inputs) - if (locale === "fr") return fr_runcolagent2(inputs) - if (locale === "uk") return uk_runcolagent2(inputs) - return en_runcolagent2(inputs) -}); -export { runcolagent2 as "runColAgent" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runcolauth2.js b/apps/web/src/paraglide/messages/runcolauth2.js deleted file mode 100644 index 6b8f8de75..000000000 --- a/apps/web/src/paraglide/messages/runcolauth2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runcolauth2Inputs */ - -const en_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Auth`) -}; - -const es_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Autenticación`) -}; - -const zh_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`身份验证`) -}; - -const ja_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`認証`) -}; - -const ko_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`인증`) -}; - -const zh_hant1_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`身份驗證`) -}; - -const de_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Authentifizierung`) -}; - -const fr_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Authentification`) -}; - -const uk_runcolauth2 = /** @type {(inputs: Runcolauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Автентифікація`) -}; - -/** -* | output | -* | --- | -* | "Auth" | -* -* @param {Runcolauth2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runcolauth2 = /** @type {((inputs?: Runcolauth2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runcolauth2(inputs) - if (locale === "zh") return zh_runcolauth2(inputs) - if (locale === "ja") return ja_runcolauth2(inputs) - if (locale === "ko") return ko_runcolauth2(inputs) - if (locale === "zh-Hant") return zh_hant1_runcolauth2(inputs) - if (locale === "de") return de_runcolauth2(inputs) - if (locale === "fr") return fr_runcolauth2(inputs) - if (locale === "uk") return uk_runcolauth2(inputs) - return en_runcolauth2(inputs) -}); -export { runcolauth2 as "runColAuth" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runcolmodels2.js b/apps/web/src/paraglide/messages/runcolmodels2.js deleted file mode 100644 index 9452d4bd3..000000000 --- a/apps/web/src/paraglide/messages/runcolmodels2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runcolmodels2Inputs */ - -const en_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Example models`) -}; - -const es_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Modelos de ejemplo`) -}; - -const zh_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`示例模型`) -}; - -const ja_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`サンプルモデル`) -}; - -const ko_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`예시 모델`) -}; - -const zh_hant1_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`範例模型`) -}; - -const de_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Beispielmodelle`) -}; - -const fr_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Exemples de modèles`) -}; - -const uk_runcolmodels2 = /** @type {(inputs: Runcolmodels2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Приклади моделей`) -}; - -/** -* | output | -* | --- | -* | "Example models" | -* -* @param {Runcolmodels2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runcolmodels2 = /** @type {((inputs?: Runcolmodels2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runcolmodels2(inputs) - if (locale === "zh") return zh_runcolmodels2(inputs) - if (locale === "ja") return ja_runcolmodels2(inputs) - if (locale === "ko") return ko_runcolmodels2(inputs) - if (locale === "zh-Hant") return zh_hant1_runcolmodels2(inputs) - if (locale === "de") return de_runcolmodels2(inputs) - if (locale === "fr") return fr_runcolmodels2(inputs) - if (locale === "uk") return uk_runcolmodels2(inputs) - return en_runcolmodels2(inputs) -}); -export { runcolmodels2 as "runColModels" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runctadesc2.js b/apps/web/src/paraglide/messages/runctadesc2.js deleted file mode 100644 index dc45ee5a9..000000000 --- a/apps/web/src/paraglide/messages/runctadesc2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runctadesc2Inputs */ - -const en_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Your numbers land in the same format as the leaderboard. Ran something interesting? Open a pull request with your report.`) -}; - -const es_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tus resultados se mostrarán en el mismo formato que la tabla de clasificación. ¿Obtuviste algo interesante? Abre un pull request con tu informe.`) -}; - -const zh_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`你的统计数据将以与排行榜相同的格式显示。运行了什么有趣的东西吗?提交一个 pull request 来附上你的报告吧。`) -}; - -const ja_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`あなたの数値はリーダーボードと同じ形式で表示されます。何か興味深い結果が出ましたか?レポートを添えてプルリクエストを作成してください。`) -}; - -const ko_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`여러분의 결과는 리더보드와 동일한 형식으로 표시됩니다. 흥미로운 작업을 수행하셨나요? 보고서를 첨부하여 풀 리퀘스트를 열어주세요.`) -}; - -const zh_hant1_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`你的統計數據將以與排行榜相同的格式顯示。運行了什麼有趣的東西嗎?提交一個 pull request 來附上你的報告吧。`) -}; - -const de_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ihre Zahlen werden im gleichen Format wie die Rangliste angezeigt. Haben Sie etwas Interessantes ausprobiert? Erstellen Sie einen Pull Request mit Ihrem Bericht.`) -}; - -const fr_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Vos résultats s'affichent au même format que le classement. Vous avez réalisé une expérience intéressante ? Soumettez une pull request avec votre rapport.`) -}; - -const uk_runctadesc2 = /** @type {(inputs: Runctadesc2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ваші числа мають той самий формат, що й leaderboard. Запустили щось цікаве? Відкрийте PR зі звітом.`) -}; - -/** -* | output | -* | --- | -* | "Your numbers land in the same format as the leaderboard. Ran something interesting? Open a pull request with your report." | -* -* @param {Runctadesc2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runctadesc2 = /** @type {((inputs?: Runctadesc2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runctadesc2(inputs) - if (locale === "zh") return zh_runctadesc2(inputs) - if (locale === "ja") return ja_runctadesc2(inputs) - if (locale === "ko") return ko_runctadesc2(inputs) - if (locale === "zh-Hant") return zh_hant1_runctadesc2(inputs) - if (locale === "de") return de_runctadesc2(inputs) - if (locale === "fr") return fr_runctadesc2(inputs) - if (locale === "uk") return uk_runctadesc2(inputs) - return en_runctadesc2(inputs) -}); -export { runctadesc2 as "runCtaDesc" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runctaeyebrow2.js b/apps/web/src/paraglide/messages/runctaeyebrow2.js deleted file mode 100644 index 340704d13..000000000 --- a/apps/web/src/paraglide/messages/runctaeyebrow2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runctaeyebrow2Inputs */ - -const en_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Compare`) -}; - -const es_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Comparar`) -}; - -const zh_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`比较`) -}; - -const ja_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`比較する`) -}; - -const ko_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`비교하다`) -}; - -const zh_hant1_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`比較`) -}; - -const de_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Vergleichen`) -}; - -const fr_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Comparer`) -}; - -const uk_runctaeyebrow2 = /** @type {(inputs: Runctaeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Порівняйте`) -}; - -/** -* | output | -* | --- | -* | "Compare" | -* -* @param {Runctaeyebrow2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runctaeyebrow2 = /** @type {((inputs?: Runctaeyebrow2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runctaeyebrow2(inputs) - if (locale === "zh") return zh_runctaeyebrow2(inputs) - if (locale === "ja") return ja_runctaeyebrow2(inputs) - if (locale === "ko") return ko_runctaeyebrow2(inputs) - if (locale === "zh-Hant") return zh_hant1_runctaeyebrow2(inputs) - if (locale === "de") return de_runctaeyebrow2(inputs) - if (locale === "fr") return fr_runctaeyebrow2(inputs) - if (locale === "uk") return uk_runctaeyebrow2(inputs) - return en_runctaeyebrow2(inputs) -}); -export { runctaeyebrow2 as "runCtaEyebrow" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runctaleaderboard2.js b/apps/web/src/paraglide/messages/runctaleaderboard2.js deleted file mode 100644 index d4c3b7ef5..000000000 --- a/apps/web/src/paraglide/messages/runctaleaderboard2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runctaleaderboard2Inputs */ - -const en_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`View the leaderboard`) -}; - -const es_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ver la tabla de clasificación`) -}; - -const zh_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`查看排行榜`) -}; - -const ja_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`リーダーボードを見る`) -}; - -const ko_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`순위표를 확인하세요`) -}; - -const zh_hant1_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`看排行榜`) -}; - -const de_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Rangliste ansehen`) -}; - -const fr_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Consultez le classement`) -}; - -const uk_runctaleaderboard2 = /** @type {(inputs: Runctaleaderboard2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Переглянути leaderboard`) -}; - -/** -* | output | -* | --- | -* | "View the leaderboard" | -* -* @param {Runctaleaderboard2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runctaleaderboard2 = /** @type {((inputs?: Runctaleaderboard2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runctaleaderboard2(inputs) - if (locale === "zh") return zh_runctaleaderboard2(inputs) - if (locale === "ja") return ja_runctaleaderboard2(inputs) - if (locale === "ko") return ko_runctaleaderboard2(inputs) - if (locale === "zh-Hant") return zh_hant1_runctaleaderboard2(inputs) - if (locale === "de") return de_runctaleaderboard2(inputs) - if (locale === "fr") return fr_runctaleaderboard2(inputs) - if (locale === "uk") return uk_runctaleaderboard2(inputs) - return en_runctaleaderboard2(inputs) -}); -export { runctaleaderboard2 as "runCtaLeaderboard" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runctatitle2.js b/apps/web/src/paraglide/messages/runctatitle2.js deleted file mode 100644 index a4e5ef1c5..000000000 --- a/apps/web/src/paraglide/messages/runctatitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runctatitle2Inputs */ - -const en_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`See how your run stacks up`) -}; - -const es_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Mira cómo se compara tu ejecución`) -}; - -const zh_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`看看你的成绩如何`) -}; - -const ja_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`あなたの実行結果がどれだけ通用するか見てみましょう`) -}; - -const ko_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`내 기록이 다른 기록들과 어떻게 다른지 확인해 보세요.`) -}; - -const zh_hant1_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`看看你的成績如何`) -}; - -const de_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Sehen Sie, wie Ihr Durchlauf abschneidet`) -}; - -const fr_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Voyez comment votre exécution se compare.`) -}; - -const uk_runctatitle2 = /** @type {(inputs: Runctatitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Подивіться, як виглядає ваш прогін`) -}; - -/** -* | output | -* | --- | -* | "See how your run stacks up" | -* -* @param {Runctatitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runctatitle2 = /** @type {((inputs?: Runctatitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runctatitle2(inputs) - if (locale === "zh") return zh_runctatitle2(inputs) - if (locale === "ja") return ja_runctatitle2(inputs) - if (locale === "ko") return ko_runctatitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_runctatitle2(inputs) - if (locale === "de") return de_runctatitle2(inputs) - if (locale === "fr") return fr_runctatitle2(inputs) - if (locale === "uk") return uk_runctatitle2(inputs) - return en_runctatitle2(inputs) -}); -export { runctatitle2 as "runCtaTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagefforts2.js b/apps/web/src/paraglide/messages/runflagefforts2.js deleted file mode 100644 index 02f3fc01a..000000000 --- a/apps/web/src/paraglide/messages/runflagefforts2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagefforts2Inputs */ - -const en_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`reasoning effort, where the model supports it`) -}; - -const es_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`esfuerzo de razonamiento, cuando el modelo lo admite`) -}; - -const zh_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`推理强度,在模型支持的情况下`) -}; - -const ja_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`モデルがそれをサポートする推論努力`) -}; - -const ko_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`모델이 뒷받침하는 추론 노력`) -}; - -const zh_hant1_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`推理強度(在模型支援的情況下)`) -}; - -const de_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Argumentationsaufwand, sofern das Modell dies unterstützt`) -}; - -const fr_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`effort de raisonnement, lorsque le modèle le prend en charge`) -}; - -const uk_runflagefforts2 = /** @type {(inputs: Runflagefforts2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`міркування, якщо модель підтримує це`) -}; - -/** -* | output | -* | --- | -* | "reasoning effort, where the model supports it" | -* -* @param {Runflagefforts2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagefforts2 = /** @type {((inputs?: Runflagefforts2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagefforts2(inputs) - if (locale === "zh") return zh_runflagefforts2(inputs) - if (locale === "ja") return ja_runflagefforts2(inputs) - if (locale === "ko") return ko_runflagefforts2(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagefforts2(inputs) - if (locale === "de") return de_runflagefforts2(inputs) - if (locale === "fr") return fr_runflagefforts2(inputs) - if (locale === "uk") return uk_runflagefforts2(inputs) - return en_runflagefforts2(inputs) -}); -export { runflagefforts2 as "runFlagEfforts" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagmodel2.js b/apps/web/src/paraglide/messages/runflagmodel2.js deleted file mode 100644 index f8ea12a95..000000000 --- a/apps/web/src/paraglide/messages/runflagmodel2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagmodel2Inputs */ - -const en_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`the model to run (see the table above); the provider is inferred from the id`) -}; - -const es_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`el modelo a ejecutar (ver la tabla anterior); el proveedor se infiere del id`) -}; - -const zh_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`要运行的模型(参见上表);提供者由 ID 推断得出。`) -}; - -const ja_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`実行するモデル(上記の表を参照)。プロバイダーはIDから推測されます。`) -}; - -const ko_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`실행할 모델(위 표 참조); 공급자는 ID에서 추론됩니다.`) -}; - -const zh_hant1_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`要運行的模型(參見上表);提供者由 ID 推斷得出。`) -}; - -const de_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Das auszuführende Modell (siehe Tabelle oben); der Provider wird aus der ID abgeleitet.`) -}; - -const fr_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`le modèle à exécuter (voir le tableau ci-dessus) ; le fournisseur est déduit de l’identifiant`) -}; - -const uk_runflagmodel2 = /** @type {(inputs: Runflagmodel2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`модель для запуску (див. таблицю вище); провайдер визначається за ідентифікатором`) -}; - -/** -* | output | -* | --- | -* | "the model to run (see the table above); the provider is inferred from the id" | -* -* @param {Runflagmodel2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagmodel2 = /** @type {((inputs?: Runflagmodel2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagmodel2(inputs) - if (locale === "zh") return zh_runflagmodel2(inputs) - if (locale === "ja") return ja_runflagmodel2(inputs) - if (locale === "ko") return ko_runflagmodel2(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagmodel2(inputs) - if (locale === "de") return de_runflagmodel2(inputs) - if (locale === "fr") return fr_runflagmodel2(inputs) - if (locale === "uk") return uk_runflagmodel2(inputs) - return en_runflagmodel2(inputs) -}); -export { runflagmodel2 as "runFlagModel" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagoutdir3.js b/apps/web/src/paraglide/messages/runflagoutdir3.js deleted file mode 100644 index 7e5213894..000000000 --- a/apps/web/src/paraglide/messages/runflagoutdir3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagoutdir3Inputs */ - -const en_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`where results land; re-use the same directory to resume or validate`) -}; - -const es_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`donde se almacenan los resultados; reutilizar el mismo directorio para reanudar o validar.`) -}; - -const zh_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`结果落在哪里;重用同一目录以继续或验证`) -}; - -const ja_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`結果が保存される場所。同じディレクトリを再利用して再開または検証します。`) -}; - -const ko_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`결과가 저장되는 위치; 동일한 디렉터리를 재사용하여 작업을 재개하거나 유효성을 검사합니다.`) -}; - -const zh_hant1_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`結果落在哪裡;重複使用同一目錄以繼續或驗證`) -}; - -const de_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`wo die Ergebnisse landen; verwenden Sie dasselbe Verzeichnis wieder, um fortzufahren oder zu validieren`) -}; - -const fr_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`où les résultats sont enregistrés ; réutiliser le même répertoire pour reprendre ou valider`) -}; - -const uk_runflagoutdir3 = /** @type {(inputs: Runflagoutdir3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`куди записуються результати; використовуйте той самий каталог, щоб відновити або перевірити`) -}; - -/** -* | output | -* | --- | -* | "where results land; re-use the same directory to resume or validate" | -* -* @param {Runflagoutdir3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagoutdir3 = /** @type {((inputs?: Runflagoutdir3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagoutdir3(inputs) - if (locale === "zh") return zh_runflagoutdir3(inputs) - if (locale === "ja") return ja_runflagoutdir3(inputs) - if (locale === "ko") return ko_runflagoutdir3(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagoutdir3(inputs) - if (locale === "de") return de_runflagoutdir3(inputs) - if (locale === "fr") return fr_runflagoutdir3(inputs) - if (locale === "uk") return uk_runflagoutdir3(inputs) - return en_runflagoutdir3(inputs) -}); -export { runflagoutdir3 as "runFlagOutDir" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagpaths2.js b/apps/web/src/paraglide/messages/runflagpaths2.js deleted file mode 100644 index 9309dae9f..000000000 --- a/apps/web/src/paraglide/messages/runflagpaths2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagpaths2Inputs */ - -const en_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt hand-writes everything; mcp goes through the MCP tools; cli composes the CLI command`) -}; - -const es_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt escribe todo a mano; mcp pasa por las herramientas de MCP; cli compone el comando CLI.`) -}; - -const zh_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt 负责手动输入所有内容;mcp 负责使用 MCP 工具;cli 负责编写 CLI 命令。`) -}; - -const ja_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt はすべてを手書きで入力し、mcp は MCP ツールを経由し、cli は CLI コマンドを構成します。`) -}; - -const ko_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt는 모든 것을 직접 작성하고, mcp는 MCP 도구를 사용하며, cli는 CLI 명령어를 작성합니다.`) -}; - -const zh_hant1_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt 負責手動輸入所有內容;mcp 負責使用 MCP 工具;cli 負責編寫 CLI 指令。`) -}; - -const de_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Die Eingabeaufforderung schreibt alles manuell; MCP durchläuft die MCP-Tools; CLI setzt den CLI-Befehl zusammen.`) -}; - -const fr_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt saisit tout manuellement ; mcp parcourt les outils MCP ; cli compose la commande CLI`) -}; - -const uk_runflagpaths2 = /** @type {(inputs: Runflagpaths2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`prompt пише все вручну; mcp проходить через MCP-інструменти; cli складає CLI-команду`) -}; - -/** -* | output | -* | --- | -* | "prompt hand-writes everything; mcp goes through the MCP tools; cli composes the CLI command" | -* -* @param {Runflagpaths2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagpaths2 = /** @type {((inputs?: Runflagpaths2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagpaths2(inputs) - if (locale === "zh") return zh_runflagpaths2(inputs) - if (locale === "ja") return ja_runflagpaths2(inputs) - if (locale === "ko") return ko_runflagpaths2(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagpaths2(inputs) - if (locale === "de") return de_runflagpaths2(inputs) - if (locale === "fr") return fr_runflagpaths2(inputs) - if (locale === "uk") return uk_runflagpaths2(inputs) - return en_runflagpaths2(inputs) -}); -export { runflagpaths2 as "runFlagPaths" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagphase2.js b/apps/web/src/paraglide/messages/runflagphase2.js deleted file mode 100644 index f9abe4c8c..000000000 --- a/apps/web/src/paraglide/messages/runflagphase2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagphase2Inputs */ - -const en_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`split the run into a generate phase and a validate phase, validated on its own`) -}; - -const es_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`dividir la ejecución en una fase de generación y una fase de validación, validada por sí sola.`) -}; - -const zh_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`将运行过程分为生成阶段和验证阶段,验证阶段单独进行验证。`) -}; - -const ja_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`実行を生成フェーズと検証フェーズに分割し、それぞれを個別に検証する`) -}; - -const ko_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`실행 과정을 생성 단계와 검증 단계로 나누고, 각 단계는 자체적으로 검증됩니다.`) -}; - -const zh_hant1_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`將運行過程分為生成階段和驗證階段,驗證階段單獨進行驗證。`) -}; - -const de_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Der Lauf wird in eine Generierungsphase und eine Validierungsphase unterteilt, die jeweils selbstständig validiert wird.`) -}; - -const fr_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`diviser l'exécution en une phase de génération et une phase de validation, validée séparément.`) -}; - -const uk_runflagphase2 = /** @type {(inputs: Runflagphase2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`розділити прогін на фазу генерації та окрему фазу перевірки`) -}; - -/** -* | output | -* | --- | -* | "split the run into a generate phase and a validate phase, validated on its own" | -* -* @param {Runflagphase2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagphase2 = /** @type {((inputs?: Runflagphase2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagphase2(inputs) - if (locale === "zh") return zh_runflagphase2(inputs) - if (locale === "ja") return ja_runflagphase2(inputs) - if (locale === "ko") return ko_runflagphase2(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagphase2(inputs) - if (locale === "de") return de_runflagphase2(inputs) - if (locale === "fr") return fr_runflagphase2(inputs) - if (locale === "uk") return uk_runflagphase2(inputs) - return en_runflagphase2(inputs) -}); -export { runflagphase2 as "runFlagPhase" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagseyebrow2.js b/apps/web/src/paraglide/messages/runflagseyebrow2.js deleted file mode 100644 index bba0c1434..000000000 --- a/apps/web/src/paraglide/messages/runflagseyebrow2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagseyebrow2Inputs */ - -const en_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Flags`) -}; - -const es_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Flags`) -}; - -const zh_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`命令行参数`) -}; - -const ja_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`フラグ`) -}; - -const ko_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`플래그`) -}; - -const zh_hant1_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`旗標`) -}; - -const de_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Flags`) -}; - -const fr_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Indicateurs`) -}; - -const uk_runflagseyebrow2 = /** @type {(inputs: Runflagseyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Прапори`) -}; - -/** -* | output | -* | --- | -* | "Flags" | -* -* @param {Runflagseyebrow2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagseyebrow2 = /** @type {((inputs?: Runflagseyebrow2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagseyebrow2(inputs) - if (locale === "zh") return zh_runflagseyebrow2(inputs) - if (locale === "ja") return ja_runflagseyebrow2(inputs) - if (locale === "ko") return ko_runflagseyebrow2(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagseyebrow2(inputs) - if (locale === "de") return de_runflagseyebrow2(inputs) - if (locale === "fr") return fr_runflagseyebrow2(inputs) - if (locale === "uk") return uk_runflagseyebrow2(inputs) - return en_runflagseyebrow2(inputs) -}); -export { runflagseyebrow2 as "runFlagsEyebrow" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagspecs2.js b/apps/web/src/paraglide/messages/runflagspecs2.js deleted file mode 100644 index dc9d758ea..000000000 --- a/apps/web/src/paraglide/messages/runflagspecs2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagspecs2Inputs */ - -const en_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`the full 13-spec suite by default, or a comma-separated subset of spec ids`) -}; - -const es_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`El conjunto completo de 13 especificaciones por defecto, o un subconjunto de identificadores de especificaciones separados por comas.`) -}; - -const zh_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`默认情况下,使用完整的 13 个规范套件;或者使用以逗号分隔的规范 ID 子集。`) -}; - -const ja_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`デフォルトでは13個の仕様すべて、またはカンマ区切りの仕様IDのサブセット`) -}; - -const ko_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`기본적으로 13개 사양 전체 또는 쉼표로 구분된 사양 ID 하위 집합을 사용할 수 있습니다.`) -}; - -const zh_hant1_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`預設情況下,使用完整的 13 個規範套件;或使用以逗號分隔的規範 ID 子集。`) -}; - -const de_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Standardmäßig die vollständige Suite mit 13 Spezifikationen oder eine durch Kommas getrennte Teilmenge der Spezifikations-IDs.`) -}; - -const fr_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`par défaut, la suite complète de 13 spécifications, ou un sous-ensemble d'identifiants de spécifications séparés par des virgules.`) -}; - -const uk_runflagspecs2 = /** @type {(inputs: Runflagspecs2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`повний набір із 13 специфікацій за замовчуванням або підмножина специфікацій, розділених комами`) -}; - -/** -* | output | -* | --- | -* | "the full 13-spec suite by default, or a comma-separated subset of spec ids" | -* -* @param {Runflagspecs2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagspecs2 = /** @type {((inputs?: Runflagspecs2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagspecs2(inputs) - if (locale === "zh") return zh_runflagspecs2(inputs) - if (locale === "ja") return ja_runflagspecs2(inputs) - if (locale === "ko") return ko_runflagspecs2(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagspecs2(inputs) - if (locale === "de") return de_runflagspecs2(inputs) - if (locale === "fr") return fr_runflagspecs2(inputs) - if (locale === "uk") return uk_runflagspecs2(inputs) - return en_runflagspecs2(inputs) -}); -export { runflagspecs2 as "runFlagSpecs" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runflagstitle2.js b/apps/web/src/paraglide/messages/runflagstitle2.js deleted file mode 100644 index 0626140c9..000000000 --- a/apps/web/src/paraglide/messages/runflagstitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runflagstitle2Inputs */ - -const en_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tune the run`) -}; - -const es_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ajusta la ejecución`) -}; - -const zh_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`调整运行`) -}; - -const ja_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`実行の調整`) -}; - -const ko_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`실행을 조정하세요`) -}; - -const zh_hant1_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`調整運行`) -}; - -const de_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Den Lauf optimieren`) -}; - -const fr_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ajustez l'exécution`) -}; - -const uk_runflagstitle2 = /** @type {(inputs: Runflagstitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Налаштуйте запуск`) -}; - -/** -* | output | -* | --- | -* | "Tune the run" | -* -* @param {Runflagstitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runflagstitle2 = /** @type {((inputs?: Runflagstitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runflagstitle2(inputs) - if (locale === "zh") return zh_runflagstitle2(inputs) - if (locale === "ja") return ja_runflagstitle2(inputs) - if (locale === "ko") return ko_runflagstitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_runflagstitle2(inputs) - if (locale === "de") return de_runflagstitle2(inputs) - if (locale === "fr") return fr_runflagstitle2(inputs) - if (locale === "uk") return uk_runflagstitle2(inputs) - return en_runflagstitle2(inputs) -}); -export { runflagstitle2 as "runFlagsTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runherobrowsereports3.js b/apps/web/src/paraglide/messages/runherobrowsereports3.js deleted file mode 100644 index 3fb12f486..000000000 --- a/apps/web/src/paraglide/messages/runherobrowsereports3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runherobrowsereports3Inputs */ - -const en_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Browse the reports`) -}; - -const es_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Consultar los informes`) -}; - -const zh_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`浏览报告`) -}; - -const ja_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`レポートを閲覧する`) -}; - -const ko_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`보고서를 살펴보세요`) -}; - -const zh_hant1_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`瀏覽報告`) -}; - -const de_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Durchsuchen Sie die Berichte.`) -}; - -const fr_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Consultez les rapports`) -}; - -const uk_runherobrowsereports3 = /** @type {(inputs: Runherobrowsereports3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Перегляньте звіти`) -}; - -/** -* | output | -* | --- | -* | "Browse the reports" | -* -* @param {Runherobrowsereports3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runherobrowsereports3 = /** @type {((inputs?: Runherobrowsereports3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runherobrowsereports3(inputs) - if (locale === "zh") return zh_runherobrowsereports3(inputs) - if (locale === "ja") return ja_runherobrowsereports3(inputs) - if (locale === "ko") return ko_runherobrowsereports3(inputs) - if (locale === "zh-Hant") return zh_hant1_runherobrowsereports3(inputs) - if (locale === "de") return de_runherobrowsereports3(inputs) - if (locale === "fr") return fr_runherobrowsereports3(inputs) - if (locale === "uk") return uk_runherobrowsereports3(inputs) - return en_runherobrowsereports3(inputs) -}); -export { runherobrowsereports3 as "runHeroBrowseReports" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runherodescription2.js b/apps/web/src/paraglide/messages/runherodescription2.js deleted file mode 100644 index f4b154477..000000000 --- a/apps/web/src/paraglide/messages/runherodescription2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runherodescription2Inputs */ - -const en_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`The harness is open source. Clone it, point it at any agent - Claude Code, Codex, opencode, Kilo, or Antigravity for Gemini - and it scaffolds each spec, then scores whether the generated project actually installs and builds. Runs work with a logged-in CLI or a plain API key.`) -}; - -const es_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`El harness es de código abierto. Clónalo, apúntalo a cualquier agente (Claude Code, Codex, opencode, Kilo o Antigravity para Gemini) y generará la estructura de cada especificación, luego verificará si el proyecto generado realmente se instala y compila. Las ejecuciones funcionan con una CLI con sesión iniciada o con una simple clave API.`) -}; - -const zh_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`该框架是开源的。克隆它,并将其指向任何代理--Claude Code、Codex、opencode、Kilo 或 Antigravity for Gemini--它就会为每个规范生成脚手架,然后评估生成的项目是否能够实际安装和构建。它支持使用已登录的 CLI 或纯 API 密钥运行。`) -}; - -const ja_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`このハーネスはオープンソースです。クローンを作成し、Claude Code、Codex、opencode、Kilo、またはGemini用のAntigravityといった任意のエージェントを指定すると、各仕様のひな形が生成され、生成されたプロジェクトが実際にインストールおよびビルドできるかどうかが評価されます。ログイン済みのCLIまたは通常のAPIキーを使用して作業を実行できます。`) -}; - -const ko_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`이 도구는 오픈 소스입니다. 클론하고 Claude Code, Codex, opencode, Kilo 또는 Gemini용 Antigravity와 같은 에이전트를 지정하면 각 사양에 대한 스캐폴딩을 생성하고 생성된 프로젝트가 실제로 설치 및 빌드되는지 여부를 평가합니다. 로그인한 CLI 또는 일반 API 키를 사용하여 작업을 실행할 수 있습니다.`) -}; - -const zh_hant1_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`該框架是開源的。複製它,並將其指向任何代理程式--Claude Code、Codex、opencode、Kilo 或 Antigravity for Gemini--它就會為每個規範生成腳手架,然後評估生成的專案是否能夠實際安裝和建置。它支援使用已登入的 CLI 或純 API 金鑰運行。`) -}; - -const de_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Das Framework ist Open Source. Klonen Sie es, verweisen Sie es auf einen beliebigen Agenten – Claude Code, Codex, opencode, Kilo oder Antigravity für Gemini – und es generiert für jede Spezifikation ein Gerüst und prüft anschließend, ob das erstellte Projekt installiert und kompiliert werden kann. Die Ausführung erfolgt über eine angemeldete CLI oder einen einfachen API-Schlüssel.`) -}; - -const fr_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ce framework est open source. Clonez-le, configurez-le avec n'importe quel agent (Claude Code, Codex, opencode, Kilo ou Antigravity pour Gemini) et il générera la structure de chaque spécification, puis vérifiera si le projet généré s'installe et se compile correctement. Il fonctionne avec une interface de ligne de commande (CLI) ou une simple clé API.`) -}; - -const uk_runherodescription2 = /** @type {(inputs: Runherodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Harness має відкритий код. Клонуйте його, підключіть будь-якого агента - Claude Code, Codex, opencode, Kilo або Antigravity для Gemini - і він згенерує кожну специфікацію, а потім перевірить, чи встановлюється та збирається проєкт. Працює з авторизованим CLI або звичайним API-ключем.`) -}; - -/** -* | output | -* | --- | -* | "The harness is open source. Clone it, point it at any agent - Claude Code, Codex, opencode, Kilo, or Antigravity for Gemini - and it scaffolds each spec, the..." | -* -* @param {Runherodescription2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runherodescription2 = /** @type {((inputs?: Runherodescription2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runherodescription2(inputs) - if (locale === "zh") return zh_runherodescription2(inputs) - if (locale === "ja") return ja_runherodescription2(inputs) - if (locale === "ko") return ko_runherodescription2(inputs) - if (locale === "zh-Hant") return zh_hant1_runherodescription2(inputs) - if (locale === "de") return de_runherodescription2(inputs) - if (locale === "fr") return fr_runherodescription2(inputs) - if (locale === "uk") return uk_runherodescription2(inputs) - return en_runherodescription2(inputs) -}); -export { runherodescription2 as "runHeroDescription" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runheroeyebrow2.js b/apps/web/src/paraglide/messages/runheroeyebrow2.js deleted file mode 100644 index 6ec2a5919..000000000 --- a/apps/web/src/paraglide/messages/runheroeyebrow2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runheroeyebrow2Inputs */ - -const en_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproduce it`) -}; - -const es_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproducirlo`) -}; - -const zh_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`亲自重现`) -}; - -const ja_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`それを再現する`) -}; - -const ko_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`복제하세요`) -}; - -const zh_hant1_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`自行重現`) -}; - -const de_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproduziere es`) -}; - -const fr_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproduisez-le`) -}; - -const uk_runheroeyebrow2 = /** @type {(inputs: Runheroeyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Відтворіть самі`) -}; - -/** -* | output | -* | --- | -* | "Reproduce it" | -* -* @param {Runheroeyebrow2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runheroeyebrow2 = /** @type {((inputs?: Runheroeyebrow2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runheroeyebrow2(inputs) - if (locale === "zh") return zh_runheroeyebrow2(inputs) - if (locale === "ja") return ja_runheroeyebrow2(inputs) - if (locale === "ko") return ko_runheroeyebrow2(inputs) - if (locale === "zh-Hant") return zh_hant1_runheroeyebrow2(inputs) - if (locale === "de") return de_runheroeyebrow2(inputs) - if (locale === "fr") return fr_runheroeyebrow2(inputs) - if (locale === "uk") return uk_runheroeyebrow2(inputs) - return en_runheroeyebrow2(inputs) -}); -export { runheroeyebrow2 as "runHeroEyebrow" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runheroquickstart2.js b/apps/web/src/paraglide/messages/runheroquickstart2.js deleted file mode 100644 index 6422e8347..000000000 --- a/apps/web/src/paraglide/messages/runheroquickstart2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runheroquickstart2Inputs */ - -const en_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Quickstart`) -}; - -const es_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Inicio rápido`) -}; - -const zh_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`快速入门`) -}; - -const ja_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`クイックスタート`) -}; - -const ko_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`빠른 시작`) -}; - -const zh_hant1_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`快速入門`) -}; - -const de_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Schnellstart`) -}; - -const fr_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Démarrage rapide`) -}; - -const uk_runheroquickstart2 = /** @type {(inputs: Runheroquickstart2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Швидкий старт`) -}; - -/** -* | output | -* | --- | -* | "Quickstart" | -* -* @param {Runheroquickstart2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runheroquickstart2 = /** @type {((inputs?: Runheroquickstart2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runheroquickstart2(inputs) - if (locale === "zh") return zh_runheroquickstart2(inputs) - if (locale === "ja") return ja_runheroquickstart2(inputs) - if (locale === "ko") return ko_runheroquickstart2(inputs) - if (locale === "zh-Hant") return zh_hant1_runheroquickstart2(inputs) - if (locale === "de") return de_runheroquickstart2(inputs) - if (locale === "fr") return fr_runheroquickstart2(inputs) - if (locale === "uk") return uk_runheroquickstart2(inputs) - return en_runheroquickstart2(inputs) -}); -export { runheroquickstart2 as "runHeroQuickstart" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runherotitlea3.js b/apps/web/src/paraglide/messages/runherotitlea3.js deleted file mode 100644 index a70ac53ba..000000000 --- a/apps/web/src/paraglide/messages/runherotitlea3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runherotitlea3Inputs */ - -const en_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Run ScaffBench`) -}; - -const es_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ejecuta ScaffBench`) -}; - -const zh_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`运行 ScaffBench`) -}; - -const ja_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBenchを実行する`) -}; - -const ko_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench 실행`) -}; - -const zh_hant1_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`運行 ScaffBench`) -}; - -const de_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Run ScaffBench`) -}; - -const fr_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Exécuter ScaffBench`) -}; - -const uk_runherotitlea3 = /** @type {(inputs: Runherotitlea3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Запустіть ScaffBench`) -}; - -/** -* | output | -* | --- | -* | "Run ScaffBench" | -* -* @param {Runherotitlea3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runherotitlea3 = /** @type {((inputs?: Runherotitlea3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runherotitlea3(inputs) - if (locale === "zh") return zh_runherotitlea3(inputs) - if (locale === "ja") return ja_runherotitlea3(inputs) - if (locale === "ko") return ko_runherotitlea3(inputs) - if (locale === "zh-Hant") return zh_hant1_runherotitlea3(inputs) - if (locale === "de") return de_runherotitlea3(inputs) - if (locale === "fr") return fr_runherotitlea3(inputs) - if (locale === "uk") return uk_runherotitlea3(inputs) - return en_runherotitlea3(inputs) -}); -export { runherotitlea3 as "runHeroTitleA" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runherotitleb3.js b/apps/web/src/paraglide/messages/runherotitleb3.js deleted file mode 100644 index 0a0801a2f..000000000 --- a/apps/web/src/paraglide/messages/runherotitleb3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runherotitleb3Inputs */ - -const en_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`yourself`) -}; - -const es_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`tú mismo`) -}; - -const zh_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`自己试试`) -}; - -const ja_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`あなた自身`) -}; - -const ko_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`직접`) -}; - -const zh_hant1_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`你自己`) -}; - -const de_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`selbst`) -}; - -const fr_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`vous-même`) -}; - -const uk_runherotitleb3 = /** @type {(inputs: Runherotitleb3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`самостійно`) -}; - -/** -* | output | -* | --- | -* | "yourself" | -* -* @param {Runherotitleb3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runherotitleb3 = /** @type {((inputs?: Runherotitleb3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runherotitleb3(inputs) - if (locale === "zh") return zh_runherotitleb3(inputs) - if (locale === "ja") return ja_runherotitleb3(inputs) - if (locale === "ko") return ko_runherotitleb3(inputs) - if (locale === "zh-Hant") return zh_hant1_runherotitleb3(inputs) - if (locale === "de") return de_runherotitleb3(inputs) - if (locale === "fr") return fr_runherotitleb3(inputs) - if (locale === "uk") return uk_runherotitleb3(inputs) - return en_runherotitleb3(inputs) -}); -export { runherotitleb3 as "runHeroTitleB" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runlabelclone2.js b/apps/web/src/paraglide/messages/runlabelclone2.js deleted file mode 100644 index 28314457f..000000000 --- a/apps/web/src/paraglide/messages/runlabelclone2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runlabelclone2Inputs */ - -const en_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`clone the harness`) -}; - -const es_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`clonar el harness`) -}; - -const zh_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`克隆测试框架`) -}; - -const ja_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ハーネスをクローンする`) -}; - -const ko_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`하네스를 복제하세요`) -}; - -const zh_hant1_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`克隆測試框架`) -}; - -const de_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Klonen Sie das Harness`) -}; - -const fr_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`cloner le harnais`) -}; - -const uk_runlabelclone2 = /** @type {(inputs: Runlabelclone2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`клонувати harness`) -}; - -/** -* | output | -* | --- | -* | "clone the harness" | -* -* @param {Runlabelclone2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runlabelclone2 = /** @type {((inputs?: Runlabelclone2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runlabelclone2(inputs) - if (locale === "zh") return zh_runlabelclone2(inputs) - if (locale === "ja") return ja_runlabelclone2(inputs) - if (locale === "ko") return ko_runlabelclone2(inputs) - if (locale === "zh-Hant") return zh_hant1_runlabelclone2(inputs) - if (locale === "de") return de_runlabelclone2(inputs) - if (locale === "fr") return fr_runlabelclone2(inputs) - if (locale === "uk") return uk_runlabelclone2(inputs) - return en_runlabelclone2(inputs) -}); -export { runlabelclone2 as "runLabelClone" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runlabelexportkey3.js b/apps/web/src/paraglide/messages/runlabelexportkey3.js deleted file mode 100644 index b799086c7..000000000 --- a/apps/web/src/paraglide/messages/runlabelexportkey3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runlabelexportkey3Inputs */ - -const en_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`export a provider key`) -}; - -const es_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`exportar una clave de proveedor`) -}; - -const zh_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`导出提供商密钥`) -}; - -const ja_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`プロバイダーキーをエクスポートする`) -}; - -const ko_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`공급자 키를 내보내기`) -}; - -const zh_hant1_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`導出提供者金鑰`) -}; - -const de_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`einen Anbieterschlüssel exportieren`) -}; - -const fr_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`exporter une clé de fournisseur`) -}; - -const uk_runlabelexportkey3 = /** @type {(inputs: Runlabelexportkey3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`експортувати ключ провайдера`) -}; - -/** -* | output | -* | --- | -* | "export a provider key" | -* -* @param {Runlabelexportkey3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runlabelexportkey3 = /** @type {((inputs?: Runlabelexportkey3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runlabelexportkey3(inputs) - if (locale === "zh") return zh_runlabelexportkey3(inputs) - if (locale === "ja") return ja_runlabelexportkey3(inputs) - if (locale === "ko") return ko_runlabelexportkey3(inputs) - if (locale === "zh-Hant") return zh_hant1_runlabelexportkey3(inputs) - if (locale === "de") return de_runlabelexportkey3(inputs) - if (locale === "fr") return fr_runlabelexportkey3(inputs) - if (locale === "uk") return uk_runlabelexportkey3(inputs) - return en_runlabelexportkey3(inputs) -}); -export { runlabelexportkey3 as "runLabelExportKey" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runlabelrunall3.js b/apps/web/src/paraglide/messages/runlabelrunall3.js deleted file mode 100644 index 188805569..000000000 --- a/apps/web/src/paraglide/messages/runlabelrunall3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runlabelrunall3Inputs */ - -const en_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`run all 13 specs, prompt path`) -}; - -const es_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ejecutar las 13 especificaciones, ruta prompt`) -}; - -const zh_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`运行所有 13 个规范,prompt 路径`) -}; - -const ja_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`全13仕様を実行(プロンプトパス)`) -}; - -const ko_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`13개 사양을 모두 실행하고 프롬프트 경로를 지정합니다.`) -}; - -const zh_hant1_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`運行所有 13 個測試案例,提示路徑`) -}; - -const de_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Führe alle 13 Spezifikationen aus, Eingabeaufforderungspfad`) -}; - -const fr_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`exécuter les 13 spécifications, chemin d'invite`) -}; - -const uk_runlabelrunall3 = /** @type {(inputs: Runlabelrunall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`запустити всі 13 специфікацій, шлях prompt`) -}; - -/** -* | output | -* | --- | -* | "run all 13 specs, prompt path" | -* -* @param {Runlabelrunall3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runlabelrunall3 = /** @type {((inputs?: Runlabelrunall3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runlabelrunall3(inputs) - if (locale === "zh") return zh_runlabelrunall3(inputs) - if (locale === "ja") return ja_runlabelrunall3(inputs) - if (locale === "ko") return ko_runlabelrunall3(inputs) - if (locale === "zh-Hant") return zh_hant1_runlabelrunall3(inputs) - if (locale === "de") return de_runlabelrunall3(inputs) - if (locale === "fr") return fr_runlabelrunall3(inputs) - if (locale === "uk") return uk_runlabelrunall3(inputs) - return en_runlabelrunall3(inputs) -}); -export { runlabelrunall3 as "runLabelRunAll" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runlabelsignin2.js b/apps/web/src/paraglide/messages/runlabelsignin2.js deleted file mode 100644 index cfe18367f..000000000 --- a/apps/web/src/paraglide/messages/runlabelsignin2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runlabelsignin2Inputs */ - -const en_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`sign in to your agent`) -}; - -const es_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`iniciar sesión en tu agente`) -}; - -const zh_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`登录你的代理`) -}; - -const ja_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`エージェントにサインイン`) -}; - -const ko_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`에이전트에 로그인하세요`) -}; - -const zh_hant1_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`登入您的代理`) -}; - -const de_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Melden Sie sich bei Ihrem Agenten an.`) -}; - -const fr_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Connectez-vous à votre agent`) -}; - -const uk_runlabelsignin2 = /** @type {(inputs: Runlabelsignin2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`увійдіть у свого агента`) -}; - -/** -* | output | -* | --- | -* | "sign in to your agent" | -* -* @param {Runlabelsignin2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runlabelsignin2 = /** @type {((inputs?: Runlabelsignin2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runlabelsignin2(inputs) - if (locale === "zh") return zh_runlabelsignin2(inputs) - if (locale === "ja") return ja_runlabelsignin2(inputs) - if (locale === "ko") return ko_runlabelsignin2(inputs) - if (locale === "zh-Hant") return zh_hant1_runlabelsignin2(inputs) - if (locale === "de") return de_runlabelsignin2(inputs) - if (locale === "fr") return fr_runlabelsignin2(inputs) - if (locale === "uk") return uk_runlabelsignin2(inputs) - return en_runlabelsignin2(inputs) -}); -export { runlabelsignin2 as "runLabelSignin" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runlabeltwophase3.js b/apps/web/src/paraglide/messages/runlabeltwophase3.js deleted file mode 100644 index 8d8ec5066..000000000 --- a/apps/web/src/paraglide/messages/runlabeltwophase3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runlabeltwophase3Inputs */ - -const en_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`two-phase`) -}; - -const es_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`dos fases`) -}; - -const zh_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`两阶段`) -}; - -const ja_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2フェーズ`) -}; - -const ko_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`2단계`) -}; - -const zh_hant1_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`兩階段`) -}; - -const de_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`zweiphasig`) -}; - -const fr_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`en deux phases`) -}; - -const uk_runlabeltwophase3 = /** @type {(inputs: Runlabeltwophase3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`двофазний`) -}; - -/** -* | output | -* | --- | -* | "two-phase" | -* -* @param {Runlabeltwophase3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runlabeltwophase3 = /** @type {((inputs?: Runlabeltwophase3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runlabeltwophase3(inputs) - if (locale === "zh") return zh_runlabeltwophase3(inputs) - if (locale === "ja") return ja_runlabeltwophase3(inputs) - if (locale === "ko") return ko_runlabeltwophase3(inputs) - if (locale === "zh-Hant") return zh_hant1_runlabeltwophase3(inputs) - if (locale === "de") return de_runlabeltwophase3(inputs) - if (locale === "fr") return fr_runlabeltwophase3(inputs) - if (locale === "uk") return uk_runlabeltwophase3(inputs) - return en_runlabeltwophase3(inputs) -}); -export { runlabeltwophase3 as "runLabelTwoPhase" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runquickstarteyebrow2.js b/apps/web/src/paraglide/messages/runquickstarteyebrow2.js deleted file mode 100644 index ef495fdaa..000000000 --- a/apps/web/src/paraglide/messages/runquickstarteyebrow2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runquickstarteyebrow2Inputs */ - -const en_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Quickstart`) -}; - -const es_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Inicio rápido`) -}; - -const zh_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`快速入门`) -}; - -const ja_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`クイックスタート`) -}; - -const ko_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`빠른 시작`) -}; - -const zh_hant1_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`快速入門`) -}; - -const de_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Schnellstart`) -}; - -const fr_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Démarrage rapide`) -}; - -const uk_runquickstarteyebrow2 = /** @type {(inputs: Runquickstarteyebrow2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Швидкий старт`) -}; - -/** -* | output | -* | --- | -* | "Quickstart" | -* -* @param {Runquickstarteyebrow2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runquickstarteyebrow2 = /** @type {((inputs?: Runquickstarteyebrow2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runquickstarteyebrow2(inputs) - if (locale === "zh") return zh_runquickstarteyebrow2(inputs) - if (locale === "ja") return ja_runquickstarteyebrow2(inputs) - if (locale === "ko") return ko_runquickstarteyebrow2(inputs) - if (locale === "zh-Hant") return zh_hant1_runquickstarteyebrow2(inputs) - if (locale === "de") return de_runquickstarteyebrow2(inputs) - if (locale === "fr") return fr_runquickstarteyebrow2(inputs) - if (locale === "uk") return uk_runquickstarteyebrow2(inputs) - return en_runquickstarteyebrow2(inputs) -}); -export { runquickstarteyebrow2 as "runQuickstartEyebrow" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runquickstarttitle2.js b/apps/web/src/paraglide/messages/runquickstarttitle2.js deleted file mode 100644 index eb404b04e..000000000 --- a/apps/web/src/paraglide/messages/runquickstarttitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runquickstarttitle2Inputs */ - -const en_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Three steps`) -}; - -const es_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Tres pasos`) -}; - -const zh_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`三步`) -}; - -const ja_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`3つのステップ`) -}; - -const ko_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`세 단계`) -}; - -const zh_hant1_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`三步`) -}; - -const de_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Drei Schritte`) -}; - -const fr_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Trois étapes`) -}; - -const uk_runquickstarttitle2 = /** @type {(inputs: Runquickstarttitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Три кроки`) -}; - -/** -* | output | -* | --- | -* | "Three steps" | -* -* @param {Runquickstarttitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runquickstarttitle2 = /** @type {((inputs?: Runquickstarttitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runquickstarttitle2(inputs) - if (locale === "zh") return zh_runquickstarttitle2(inputs) - if (locale === "ja") return ja_runquickstarttitle2(inputs) - if (locale === "ko") return ko_runquickstarttitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_runquickstarttitle2(inputs) - if (locale === "de") return de_runquickstarttitle2(inputs) - if (locale === "fr") return fr_runquickstarttitle2(inputs) - if (locale === "uk") return uk_runquickstarttitle2(inputs) - return en_runquickstarttitle2(inputs) -}); -export { runquickstarttitle2 as "runQuickstartTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runresultsnotelink3.js b/apps/web/src/paraglide/messages/runresultsnotelink3.js deleted file mode 100644 index c8d08531f..000000000 --- a/apps/web/src/paraglide/messages/runresultsnotelink3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runresultsnotelink3Inputs */ - -const en_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`published reports`) -}; - -const es_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`informes publicados`) -}; - -const zh_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`已发布的报告`) -}; - -const ja_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`公表された報告書`) -}; - -const ko_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`발표된 보고서`) -}; - -const zh_hant1_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`已發表的報告`) -}; - -const de_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`veröffentlichte Berichte`) -}; - -const fr_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`rapports publiés`) -}; - -const uk_runresultsnotelink3 = /** @type {(inputs: Runresultsnotelink3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`опубліковані звіти`) -}; - -/** -* | output | -* | --- | -* | "published reports" | -* -* @param {Runresultsnotelink3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runresultsnotelink3 = /** @type {((inputs?: Runresultsnotelink3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runresultsnotelink3(inputs) - if (locale === "zh") return zh_runresultsnotelink3(inputs) - if (locale === "ja") return ja_runresultsnotelink3(inputs) - if (locale === "ko") return ko_runresultsnotelink3(inputs) - if (locale === "zh-Hant") return zh_hant1_runresultsnotelink3(inputs) - if (locale === "de") return de_runresultsnotelink3(inputs) - if (locale === "fr") return fr_runresultsnotelink3(inputs) - if (locale === "uk") return uk_runresultsnotelink3(inputs) - return en_runresultsnotelink3(inputs) -}); -export { runresultsnotelink3 as "runResultsNoteLink" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runresultsnotepre3.js b/apps/web/src/paraglide/messages/runresultsnotepre3.js deleted file mode 100644 index f508c5e48..000000000 --- a/apps/web/src/paraglide/messages/runresultsnotepre3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runresultsnotepre3Inputs */ - -const en_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Results - a leaderboard, per-spec pass, wired-libraries, and cost - land in the output directory, in the same shape as the `) -}; - -const es_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Los resultados (una tabla de clasificación, el resultado por especificación, las librerías integradas y el coste) se guardan en el directorio de salida, con el mismo formato que los `) -}; - -const zh_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`结果--包括排行榜、每个规范的通过情况、已连接的库以及成本--都会保存到输出目录,格式参照`) -}; - -const ja_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`結果(リーダーボード、仕様ごとの合否、wired-libraries、コスト)は、出力ディレクトリに保存されます。形式は次と同じです: `) -}; - -const ko_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`결과(리더보드, 사양별 패스, 연결된 라이브러리 및 비용)는 출력 디렉터리에 다음과 같은 형식으로 저장됩니다. `) -}; - -const zh_hant1_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`結果--排行榜、各規範的通過情況、已串接的函式庫以及成本--都會落在輸出目錄中,格式比照 `) -}; - -const de_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Die Ergebnisse – eine Rangliste, die Ergebnisse pro Spezifikation, die verwendeten Bibliotheken und die Kosten – landen im Ausgabeverzeichnis, in der gleichen Struktur wie die `) -}; - -const fr_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Les résultats - un classement, un passage par spécification, les bibliothèques câblées et le coût - sont enregistrés dans le répertoire de sortie, sous la même forme que le `) -}; - -const uk_runresultsnotepre3 = /** @type {(inputs: Runresultsnotepre3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Результати - leaderboard, проходження за специфікаціями, підключені бібліотеки й вартість - потрапляють у вихідний каталог у тому самому форматі, що й`) -}; - -/** -* | output | -* | --- | -* | "Results - a leaderboard, per-spec pass, wired-libraries, and cost - land in the output directory, in the same shape as the" | -* -* @param {Runresultsnotepre3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runresultsnotepre3 = /** @type {((inputs?: Runresultsnotepre3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runresultsnotepre3(inputs) - if (locale === "zh") return zh_runresultsnotepre3(inputs) - if (locale === "ja") return ja_runresultsnotepre3(inputs) - if (locale === "ko") return ko_runresultsnotepre3(inputs) - if (locale === "zh-Hant") return zh_hant1_runresultsnotepre3(inputs) - if (locale === "de") return de_runresultsnotepre3(inputs) - if (locale === "fr") return fr_runresultsnotepre3(inputs) - if (locale === "uk") return uk_runresultsnotepre3(inputs) - return en_runresultsnotepre3(inputs) -}); -export { runresultsnotepre3 as "runResultsNotePre" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runseodescription2.js b/apps/web/src/paraglide/messages/runseodescription2.js deleted file mode 100644 index ec2de0d28..000000000 --- a/apps/web/src/paraglide/messages/runseodescription2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runseodescription2Inputs */ - -const en_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproduce the ScaffBench benchmark locally: clone the harness, point it at any agent CLI or an API key, and score whether the generated projects build.`) -}; - -const es_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproduce localmente el benchmark ScaffBench: clona el harness, apúntalo a cualquier CLI de agente o a una clave API y evalúa si los proyectos generados compilan.`) -}; - -const zh_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`在本地重现 ScaffBench 基准测试:克隆测试框架,将其指向任何代理 CLI 或 API 密钥,并评估生成的项目是否能够构建。`) -}; - -const ja_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBenchベンチマークをローカルで再現するには、ハーネスをクローンし、任意のエージェントCLIまたはAPIキーを指定して、生成されたプロジェクトがビルドされるかどうかをスコアリングします。`) -}; - -const ko_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench 벤치마크를 로컬에서 재현하려면 하네스를 복제하고, 에이전트 CLI 또는 API 키를 지정한 다음, 생성된 프로젝트가 빌드되는지 여부를 평가하십시오.`) -}; - -const zh_hant1_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`在本地重現 ScaffBench 基準測試:克隆測試框架,將其指向任何代理 CLI 或 API 金鑰,並評估產生的專案是否能夠建置。`) -}; - -const de_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproduzieren Sie den ScaffBench-Benchmark lokal: Klonen Sie das Harness, verweisen Sie es auf eine beliebige Agenten-CLI oder einen API-Schlüssel und bewerten Sie, ob die generierten Projekte kompiliert werden können.`) -}; - -const fr_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Reproduisez localement le benchmark ScaffBench : clonez le framework, pointez-le vers n’importe quelle interface de ligne de commande d’agent ou une clé API, et vérifiez si les projets générés sont compilés.`) -}; - -const uk_runseodescription2 = /** @type {(inputs: Runseodescription2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Відтворіть ScaffBench локально: клонуйте harness, підключіть будь-який agent CLI або API-ключ і перевірте, чи збираються згенеровані проєкти.`) -}; - -/** -* | output | -* | --- | -* | "Reproduce the ScaffBench benchmark locally: clone the harness, point it at any agent CLI or an API key, and score whether the generated projects build." | -* -* @param {Runseodescription2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runseodescription2 = /** @type {((inputs?: Runseodescription2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runseodescription2(inputs) - if (locale === "zh") return zh_runseodescription2(inputs) - if (locale === "ja") return ja_runseodescription2(inputs) - if (locale === "ko") return ko_runseodescription2(inputs) - if (locale === "zh-Hant") return zh_hant1_runseodescription2(inputs) - if (locale === "de") return de_runseodescription2(inputs) - if (locale === "fr") return fr_runseodescription2(inputs) - if (locale === "uk") return uk_runseodescription2(inputs) - return en_runseodescription2(inputs) -}); -export { runseodescription2 as "runSeoDescription" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runseotitle2.js b/apps/web/src/paraglide/messages/runseotitle2.js deleted file mode 100644 index d5bb3c22c..000000000 --- a/apps/web/src/paraglide/messages/runseotitle2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runseotitle2Inputs */ - -const en_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Run ScaffBench yourself`) -}; - -const es_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ejecuta ScaffBench tú mismo`) -}; - -const zh_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`自行运行 ScaffBench`) -}; - -const ja_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBenchを自分で実行してみましょう`) -}; - -const ko_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ScaffBench를 직접 실행해 보세요.`) -}; - -const zh_hant1_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`自行運行 ScaffBench`) -}; - -const de_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Betreibe ScaffBench selbst`) -}; - -const fr_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Exécutez ScaffBench vous-même`) -}; - -const uk_runseotitle2 = /** @type {(inputs: Runseotitle2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Запустіть ScaffBench самостійно`) -}; - -/** -* | output | -* | --- | -* | "Run ScaffBench yourself" | -* -* @param {Runseotitle2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runseotitle2 = /** @type {((inputs?: Runseotitle2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runseotitle2(inputs) - if (locale === "zh") return zh_runseotitle2(inputs) - if (locale === "ja") return ja_runseotitle2(inputs) - if (locale === "ko") return ko_runseotitle2(inputs) - if (locale === "zh-Hant") return zh_hant1_runseotitle2(inputs) - if (locale === "de") return de_runseotitle2(inputs) - if (locale === "fr") return fr_runseotitle2(inputs) - if (locale === "uk") return uk_runseotitle2(inputs) - return en_runseotitle2(inputs) -}); -export { runseotitle2 as "runSeoTitle" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runstepauth2.js b/apps/web/src/paraglide/messages/runstepauth2.js deleted file mode 100644 index e7dfeb629..000000000 --- a/apps/web/src/paraglide/messages/runstepauth2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runstepauth2Inputs */ - -const en_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Authenticate your agent`) -}; - -const es_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Autentica a tu agente`) -}; - -const zh_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`认证你的代理`) -}; - -const ja_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`エージェントを認証する`) -}; - -const ko_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`에이전트를 인증하세요`) -}; - -const zh_hant1_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`驗證你的代理程式`) -}; - -const de_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Authentifizieren Sie Ihren Agenten`) -}; - -const fr_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Authentifiez votre agent`) -}; - -const uk_runstepauth2 = /** @type {(inputs: Runstepauth2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Авторизуйте агента`) -}; - -/** -* | output | -* | --- | -* | "Authenticate your agent" | -* -* @param {Runstepauth2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runstepauth2 = /** @type {((inputs?: Runstepauth2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runstepauth2(inputs) - if (locale === "zh") return zh_runstepauth2(inputs) - if (locale === "ja") return ja_runstepauth2(inputs) - if (locale === "ko") return ko_runstepauth2(inputs) - if (locale === "zh-Hant") return zh_hant1_runstepauth2(inputs) - if (locale === "de") return de_runstepauth2(inputs) - if (locale === "fr") return fr_runstepauth2(inputs) - if (locale === "uk") return uk_runstepauth2(inputs) - return en_runstepauth2(inputs) -}); -export { runstepauth2 as "runStepAuth" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runstepcloneinstall3.js b/apps/web/src/paraglide/messages/runstepcloneinstall3.js deleted file mode 100644 index 6b95d59db..000000000 --- a/apps/web/src/paraglide/messages/runstepcloneinstall3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runstepcloneinstall3Inputs */ - -const en_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Clone & install`) -}; - -const es_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Clonar e instalar`) -}; - -const zh_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`克隆并安装`) -}; - -const ja_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`クローンしてインストール`) -}; - -const ko_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`복제 및 설치`) -}; - -const zh_hant1_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`克隆並安裝`) -}; - -const de_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Klonen und installieren`) -}; - -const fr_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Cloner et installer`) -}; - -const uk_runstepcloneinstall3 = /** @type {(inputs: Runstepcloneinstall3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Клонуйте та встановіть`) -}; - -/** -* | output | -* | --- | -* | "Clone & install" | -* -* @param {Runstepcloneinstall3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runstepcloneinstall3 = /** @type {((inputs?: Runstepcloneinstall3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runstepcloneinstall3(inputs) - if (locale === "zh") return zh_runstepcloneinstall3(inputs) - if (locale === "ja") return ja_runstepcloneinstall3(inputs) - if (locale === "ko") return ko_runstepcloneinstall3(inputs) - if (locale === "zh-Hant") return zh_hant1_runstepcloneinstall3(inputs) - if (locale === "de") return de_runstepcloneinstall3(inputs) - if (locale === "fr") return fr_runstepcloneinstall3(inputs) - if (locale === "uk") return uk_runstepcloneinstall3(inputs) - return en_runstepcloneinstall3(inputs) -}); -export { runstepcloneinstall3 as "runStepCloneInstall" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runsteprun2.js b/apps/web/src/paraglide/messages/runsteprun2.js deleted file mode 100644 index d4ebc9332..000000000 --- a/apps/web/src/paraglide/messages/runsteprun2.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runsteprun2Inputs */ - -const en_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Run the benchmark`) -}; - -const es_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Ejecutar el benchmark`) -}; - -const zh_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`运行基准测试`) -}; - -const ja_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`ベンチマークを実行する`) -}; - -const ko_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`벤치마크를 실행하세요`) -}; - -const zh_hant1_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`運行基準測試`) -}; - -const de_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Führen Sie den Benchmark aus.`) -}; - -const fr_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Exécuter le test de performance`) -}; - -const uk_runsteprun2 = /** @type {(inputs: Runsteprun2Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Запустіть бенчмарк`) -}; - -/** -* | output | -* | --- | -* | "Run the benchmark" | -* -* @param {Runsteprun2Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runsteprun2 = /** @type {((inputs?: Runsteprun2Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runsteprun2(inputs) - if (locale === "zh") return zh_runsteprun2(inputs) - if (locale === "ja") return ja_runsteprun2(inputs) - if (locale === "ko") return ko_runsteprun2(inputs) - if (locale === "zh-Hant") return zh_hant1_runsteprun2(inputs) - if (locale === "de") return de_runsteprun2(inputs) - if (locale === "fr") return fr_runsteprun2(inputs) - if (locale === "uk") return uk_runsteprun2(inputs) - return en_runsteprun2(inputs) -}); -export { runsteprun2 as "runStepRun" } \ No newline at end of file diff --git a/apps/web/src/paraglide/messages/runtwophasenote3.js b/apps/web/src/paraglide/messages/runtwophasenote3.js deleted file mode 100644 index 7189773a2..000000000 --- a/apps/web/src/paraglide/messages/runtwophasenote3.js +++ /dev/null @@ -1,65 +0,0 @@ -/* eslint-disable */ -import { getLocale, experimentalStaticLocale } from '../runtime.js'; - -/** @typedef {import('../runtime.js').LocalizedString} LocalizedString */ - -/** @typedef {{}} Runtwophasenote3Inputs */ - -const en_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Prefer to keep validation clean? Split it into two phases - generate everything first, then validate on its own:`) -}; - -const es_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`¿Prefieres mantener la validación limpia? Divídela en dos fases: primero genera todo y luego valida por separado.`) -}; - -const zh_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`想要保持验证过程的简洁性?那就把它分成两个阶段--先生成所有内容,然后再单独进行验证:`) -}; - -const ja_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`検証を簡潔に保ちたい場合は、2つのフェーズに分割します。まずすべてを生成し、次に検証を単独で行います。`) -}; - -const ko_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`검증 과정을 깔끔하게 유지하고 싶으신가요? 그렇다면 두 단계로 나누세요. 먼저 모든 것을 생성한 다음, 생성된 부분만 따로 검증하는 방식입니다.`) -}; - -const zh_hant1_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`想要保持驗證過程的簡潔性?那就把它分成兩個階段──先生成所有內容,然後再單獨驗證:`) -}; - -const de_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Sie möchten die Validierung übersichtlich halten? Teilen Sie sie in zwei Phasen auf – generieren Sie zuerst alles und validieren Sie anschließend separat:`) -}; - -const fr_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Vous préférez une validation simple ? Divisez-la en deux phases : générez d’abord tout, puis validez séparément :`) -}; - -const uk_runtwophasenote3 = /** @type {(inputs: Runtwophasenote3Inputs) => LocalizedString} */ () => { - return /** @type {LocalizedString} */ (`Хочете тримати валідацію чистою? Розділіть запуск на дві фази: спочатку згенеруйте все, потім перевірте окремо:`) -}; - -/** -* | output | -* | --- | -* | "Prefer to keep validation clean? Split it into two phases - generate everything first, then validate on its own:" | -* -* @param {Runtwophasenote3Inputs} inputs -* @param {{ locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }} options -* @returns {LocalizedString} -*/ -const runtwophasenote3 = /** @type {((inputs?: Runtwophasenote3Inputs, options?: { locale?: "en" | "es" | "zh" | "ja" | "ko" | "zh-Hant" | "de" | "fr" | "uk" }) => LocalizedString) & import('../runtime.js').MessageMetadata} */ ((inputs = {}, options = {}) => { - const locale = experimentalStaticLocale ?? options.locale ?? getLocale() - if (locale === "es") return es_runtwophasenote3(inputs) - if (locale === "zh") return zh_runtwophasenote3(inputs) - if (locale === "ja") return ja_runtwophasenote3(inputs) - if (locale === "ko") return ko_runtwophasenote3(inputs) - if (locale === "zh-Hant") return zh_hant1_runtwophasenote3(inputs) - if (locale === "de") return de_runtwophasenote3(inputs) - if (locale === "fr") return fr_runtwophasenote3(inputs) - if (locale === "uk") return uk_runtwophasenote3(inputs) - return en_runtwophasenote3(inputs) -}); -export { runtwophasenote3 as "runTwoPhaseNote" } \ No newline at end of file diff --git a/apps/web/src/routeTree.gen.ts b/apps/web/src/routeTree.gen.ts index 78c4ef3f0..a740fb69d 100644 --- a/apps/web/src/routeTree.gen.ts +++ b/apps/web/src/routeTree.gen.ts @@ -19,7 +19,6 @@ import { Route as LlmsFullDottxtRouteImport } from './routes/llms-full[.]txt' import { Route as LlmsDottxtRouteImport } from './routes/llms[.]txt' import { Route as McpRouteImport } from './routes/mcp' import { Route as NewRouteImport } from './routes/new' -import { Route as RunRouteImport } from './routes/run' import { Route as RunBeforeYouCloneRouteImport } from './routes/run-before-you-clone' import { Route as SitemapDotmdRouteImport } from './routes/sitemap[.]md' import { Route as SitemapDotxmlRouteImport } from './routes/sitemap[.]xml' @@ -93,11 +92,6 @@ const NewRoute = NewRouteImport.update({ path: '/new', getParentRoute: () => rootRouteImport, } as any) -const RunRoute = RunRouteImport.update({ - id: '/run', - path: '/run', - getParentRoute: () => rootRouteImport, -} as any) const RunBeforeYouCloneRoute = RunBeforeYouCloneRouteImport.update({ id: '/run-before-you-clone', path: '/run-before-you-clone', @@ -222,7 +216,6 @@ export interface FileRoutesByFullPath { '/llms.txt': typeof LlmsDottxtRoute '/mcp': typeof McpRoute '/new': typeof NewRoute - '/run': typeof RunRoute '/run-before-you-clone': typeof RunBeforeYouCloneRoute '/sitemap.md': typeof SitemapDotmdRoute '/sitemap.xml': typeof SitemapDotxmlRoute @@ -257,7 +250,6 @@ export interface FileRoutesByTo { '/llms.txt': typeof LlmsDottxtRoute '/mcp': typeof McpRoute '/new': typeof NewRoute - '/run': typeof RunRoute '/run-before-you-clone': typeof RunBeforeYouCloneRoute '/sitemap.md': typeof SitemapDotmdRoute '/sitemap.xml': typeof SitemapDotxmlRoute @@ -293,7 +285,6 @@ export interface FileRoutesById { '/llms.txt': typeof LlmsDottxtRoute '/mcp': typeof McpRoute '/new': typeof NewRoute - '/run': typeof RunRoute '/run-before-you-clone': typeof RunBeforeYouCloneRoute '/sitemap.md': typeof SitemapDotmdRoute '/sitemap.xml': typeof SitemapDotxmlRoute @@ -330,7 +321,6 @@ export interface FileRouteTypes { | '/llms.txt' | '/mcp' | '/new' - | '/run' | '/run-before-you-clone' | '/sitemap.md' | '/sitemap.xml' @@ -365,7 +355,6 @@ export interface FileRouteTypes { | '/llms.txt' | '/mcp' | '/new' - | '/run' | '/run-before-you-clone' | '/sitemap.md' | '/sitemap.xml' @@ -400,7 +389,6 @@ export interface FileRouteTypes { | '/llms.txt' | '/mcp' | '/new' - | '/run' | '/run-before-you-clone' | '/sitemap.md' | '/sitemap.xml' @@ -436,7 +424,6 @@ export interface RootRouteChildren { LlmsDottxtRoute: typeof LlmsDottxtRoute McpRoute: typeof McpRoute NewRoute: typeof NewRoute - RunRoute: typeof RunRoute RunBeforeYouCloneRoute: typeof RunBeforeYouCloneRoute SitemapDotmdRoute: typeof SitemapDotmdRoute SitemapDotxmlRoute: typeof SitemapDotxmlRoute @@ -533,13 +520,6 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof NewRouteImport parentRoute: typeof rootRouteImport } - '/run': { - id: '/run' - path: '/run' - fullPath: '/run' - preLoaderRoute: typeof RunRouteImport - parentRoute: typeof rootRouteImport - } '/run-before-you-clone': { id: '/run-before-you-clone' path: '/run-before-you-clone' @@ -708,7 +688,6 @@ const rootRouteChildren: RootRouteChildren = { LlmsDottxtRoute: LlmsDottxtRoute, McpRoute: McpRoute, NewRoute: NewRoute, - RunRoute: RunRoute, RunBeforeYouCloneRoute: RunBeforeYouCloneRoute, SitemapDotmdRoute: SitemapDotmdRoute, SitemapDotxmlRoute: SitemapDotxmlRoute, @@ -735,12 +714,3 @@ const rootRouteChildren: RootRouteChildren = { export const routeTree = rootRouteImport ._addFileChildren(rootRouteChildren) ._addFileTypes() - -import type { getRouter } from './router.tsx' -import type { createStart } from '@tanstack/react-start' -declare module '@tanstack/react-start' { - interface Register { - ssr: true - router: Awaited> - } -} diff --git a/apps/web/src/routes/benchmark.tsx b/apps/web/src/routes/benchmark.tsx index ae1e2f12e..b622e97db 100644 --- a/apps/web/src/routes/benchmark.tsx +++ b/apps/web/src/routes/benchmark.tsx @@ -1,53 +1,132 @@ import { createFileRoute } from "@tanstack/react-router"; +import type { FixproofBoard } from "@/components/benchmark/fixproof-data"; + +import { FixproofBoardTable } from "@/components/benchmark/fixproof-board"; +import { FixproofChart } from "@/components/benchmark/fixproof-chart"; +import { FIXPROOF_BOARD } from "@/components/benchmark/fixproof-data"; +import { gradedTaskCount } from "@/components/benchmark/fixproof-theme"; import Footer from "@/components/home/footer"; -import LLMBenchmarkSection from "@/components/home/llm-benchmark-section"; -import { - DEFAULT_OG_IMAGE_ALT, - DEFAULT_OG_IMAGE_HEIGHT, - DEFAULT_OG_IMAGE_URL, - DEFAULT_OG_IMAGE_WIDTH, - DEFAULT_ROBOTS, - DEFAULT_X_IMAGE_URL, - canonicalUrl, -} from "@/lib/seo/seo"; +import { buildPageHead } from "@/lib/seo/seo"; import { m } from "@/paraglide/messages.js"; export const Route = createFileRoute("/benchmark")({ - head: () => { - const title = m.benchmarkSeoTitle(); - const description = m.llmBenchmarkDescription(); - - return { - meta: [ - { title }, - { name: "description", content: description }, - { name: "robots", content: DEFAULT_ROBOTS }, - { property: "og:title", content: title }, - { property: "og:description", content: description }, - { property: "og:type", content: "website" }, - { property: "og:url", content: canonicalUrl("/benchmark") }, - { property: "og:image", content: DEFAULT_OG_IMAGE_URL }, - { property: "og:image:alt", content: DEFAULT_OG_IMAGE_ALT }, - { property: "og:image:width", content: String(DEFAULT_OG_IMAGE_WIDTH) }, - { property: "og:image:height", content: String(DEFAULT_OG_IMAGE_HEIGHT) }, - { name: "twitter:card", content: "summary_large_image" }, - { name: "twitter:title", content: title }, - { name: "twitter:description", content: description }, - { name: "twitter:image", content: DEFAULT_X_IMAGE_URL }, - { name: "twitter:image:alt", content: DEFAULT_OG_IMAGE_ALT }, - ], - links: [{ rel: "canonical", href: canonicalUrl("/benchmark") }], - }; - }, + head: () => + buildPageHead({ + title: m.fixproofSeoTitle(), + description: m.benchmarkDescription(), + path: "/benchmark", + }), component: BenchmarkPage, }); +const SECTION_SHELL = "mx-auto min-w-0 max-w-[1220px] px-4 py-12 sm:px-6 sm:py-16 lg:px-8"; + +const SECTION_HEADING = "font-mono text-xl font-bold tracking-[-0.02em] sm:text-2xl"; + +const MICRO_LABEL = + "font-medium uppercase tracking-[0.14em] text-[10px] text-[#71706a] dark:text-[#8f8d84]"; + +/** Nine cells, one resolved. The mark is the task set in miniature. */ +function FixproofMark({ className }: { className?: string }) { + return ( + + + + + + + + + + + + + + ); +} + +function StatusItem({ label, value }: { label: string; value: string }) { + return ( +
    + {label} + {value} +
    + ); +} + +function Masthead({ board }: { board: FixproofBoard }) { + return ( +
    +
    + +

    + {m.benchmarkTitle()} +

    +
    +

    {m.fixproofClaim()}

    +

    + {m.fixproofProvenanceSummary()} +

    +
    + + + + +
    +
    + ); +} + function BenchmarkPage() { + const board = FIXPROOF_BOARD; + return (
    - +
    +
    + +
    +
    + +
    +
    +

    + {m.fixproofBoardHeading()} +

    +

    + {m.fixproofBoardCaption()} +

    +
    + +
    +
    +
    + +
    +
    +

    + {m.fixproofChartHeading()} +

    +

    + {m.fixproofChartCaption()} +

    +
    + +
    +
    +
    +
    diff --git a/apps/web/src/routes/index.tsx b/apps/web/src/routes/index.tsx index 27062f7d9..b3229b998 100644 --- a/apps/web/src/routes/index.tsx +++ b/apps/web/src/routes/index.tsx @@ -1,6 +1,5 @@ import { createFileRoute } from "@tanstack/react-router"; -import BenchmarkTeaser from "@/components/home/benchmark-teaser"; import CombinationsSection from "@/components/home/combinations-section"; import ContributorsSection from "@/components/home/contributors-section"; import FeaturesSection from "@/components/home/features-section"; @@ -57,7 +56,6 @@ function HomePage() { - diff --git a/apps/web/src/routes/mcp.tsx b/apps/web/src/routes/mcp.tsx index 4cb6f79b2..9e50239c1 100644 --- a/apps/web/src/routes/mcp.tsx +++ b/apps/web/src/routes/mcp.tsx @@ -341,7 +341,6 @@ const STATS = [ { id: "tools", value: 7, suffix: "", fraction: false }, { id: "resources", value: 3, suffix: "", fraction: false }, { id: "options", value: 677, suffix: "", fraction: false }, - { id: "speed", value: 2.6, suffix: "×", fraction: true }, ] as const; const numberFlowTiming = { duration: 900, easing: "cubic-bezier(0.2, 0.8, 0.2, 1)" } as const; @@ -430,8 +429,6 @@ function getStatLabel(id: (typeof STATS)[number]["id"]) { return m.mcpStatReadableResources(); case "options": return m.mcpStatConfigurableOptions(); - case "speed": - return m.mcpStatFasterPromptOnly(); } } @@ -507,7 +504,7 @@ function HeroSection() {
    -
    +
    {STATS.map((stat, index) => ( ))} @@ -536,9 +533,7 @@ function StatCell({ transition={transition} className={cn( "border-border p-5 sm:p-6", - index % 2 === 0 && "border-r", - index < 2 && "border-b lg:border-b-0", - index < 3 && "lg:border-r", + index < STATS.length - 1 && "border-b sm:border-b-0 sm:border-r", )} >
    diff --git a/apps/web/src/routes/run.tsx b/apps/web/src/routes/run.tsx deleted file mode 100644 index 89fe50a05..000000000 --- a/apps/web/src/routes/run.tsx +++ /dev/null @@ -1,412 +0,0 @@ -import { createFileRoute } from "@tanstack/react-router"; -import { useCallback, useState, type CSSProperties } from "react"; -import { - TbArrowRight as ArrowRight, - TbArrowUpRight as ArrowUpRight, - TbCheck as Check, - TbCopy as Copy, -} from "react-icons/tb"; - -import Footer from "@/components/home/footer"; -import { cn } from "@/lib/platform/utils"; -import { buildPageHead, SITE_NAME } from "@/lib/seo/seo"; -import { m } from "@/paraglide/messages.js"; - -export const Route = createFileRoute("/run")({ - head: () => { - const title = `${m.runSeoTitle()} | ${SITE_NAME}`; - const description = m.runSeoDescription(); - return buildPageHead({ title, description, path: "/run" }); - }, - component: RunPage, -}); - -const ACCENT_TEXT = "text-black dark:text-[#C6E853]"; -const h1Style: CSSProperties = { fontSize: "clamp(2.3rem, 7vw, 4.5rem)", lineHeight: 0.96 }; -const h2Style: CSSProperties = { fontSize: "clamp(1.7rem, 4.5vw, 2.8rem)", lineHeight: 1 }; - -const REPO_URL = "https://github.com/Marve10s/Better-Fullstack"; -const REPORTS_URL = `${REPO_URL}/tree/main/benchmarks`; - -// Copyable command / code block. `code` is verbatim shell (never localized); -// `label` is a short localized caption. -function CodeBlock({ - label, - code, - shell = true, -}: { - label: string; - code: string; - shell?: boolean; -}) { - const [copied, setCopied] = useState(false); - const copy = useCallback(() => { - navigator.clipboard.writeText(code).then( - () => { - setCopied(true); - window.setTimeout(() => setCopied(false), 1600); - return; - }, - () => {}, - ); - }, [code]); - - return ( -
    -
    - - {label} - - -
    -
    -        
    -          {code.split("\n").map((line, i) => (
    -            
    -              {shell && line && !line.startsWith("#") ? (
    -                $ 
    -              ) : null}
    -              
    -                {line || " "}
    -              
    -            
    -          ))}
    -        
    -      
    -
    - ); -} - -type AuthMode = "cli" | "api"; - -function AuthPanel() { - const [mode, setMode] = useState("cli"); - return ( -
    -
    - - -
    - -
    - {mode === "cli" ? ( - <> -

    {m.runAuthCliDesc()}

    - - - ) : ( - <> -

    {m.runAuthApiDesc()}

    - - - )} -
    -
    - ); -} - -function ModeTab({ - mode, - active, - label, - onSelect, -}: { - mode: AuthMode; - active: boolean; - label: string; - onSelect: (mode: AuthMode) => void; -}) { - const handleClick = useCallback(() => onSelect(mode), [onSelect, mode]); - return ( - - ); -} - -// Technical reference - model ids / auth commands are code, kept verbatim. -const AGENTS: ReadonlyArray<{ agent: string; models: string; auth: string }> = [ - { - agent: "Claude Code", - models: "claude-opus-4-8, claude-sonnet-5, claude-sonnet-4-6", - auth: "subscription · ANTHROPIC_API_KEY", - }, - { agent: "Codex", models: "gpt-5.5, gpt-5.3-codex-spark", auth: "OPENAI_API_KEY" }, - { - agent: "Antigravity (agy)", - models: "gemini-3.5-flash, gemini-3.1-pro", - auth: "Google sign-in", - }, - { agent: "opencode", models: "opencode/ (incl. free tier)", auth: "opencode login" }, - { agent: "Kilo Code", models: "kilo// (incl. free tier)", auth: "kilo login" }, -]; - -type FlagId = "model" | "efforts" | "paths" | "specs" | "phase" | "outDir"; -const FLAGS: ReadonlyArray<{ flag: string; id: FlagId }> = [ - { flag: "--model ", id: "model" }, - { flag: "--efforts ", id: "efforts" }, - { flag: "--paths prompt|mcp|cli", id: "paths" }, - { flag: "--specs core", id: "specs" }, - { flag: "--generate-only / --validate-existing", id: "phase" }, - { flag: "--out-dir ", id: "outDir" }, -]; - -function flagDesc(id: FlagId): string { - switch (id) { - case "model": - return m.runFlagModel(); - case "efforts": - return m.runFlagEfforts(); - case "paths": - return m.runFlagPaths(); - case "specs": - return m.runFlagSpecs(); - case "phase": - return m.runFlagPhase(); - case "outDir": - return m.runFlagOutDir(); - } -} - -function RunPage() { - return ( -
    -
    - {/* Hero */} -
    -
    -

    - ✦ {m.runHeroEyebrow()} -

    -

    - {m.runHeroTitleA()}{" "} - {m.runHeroTitleB()} -

    -

    - {m.runHeroDescription()} -

    - -
    -
    - - {/* Quickstart */} -
    -
    -

    - ✦ {m.runQuickstartEyebrow()} -

    -

    - {m.runQuickstartTitle()} -

    - -
      -
    1. - -
      - -
      -
    2. -
    3. - -
      - -
      -
    4. -
    5. - -
      - -

      {m.runTwoPhaseNote()}

      - -

      - {m.runResultsNotePre()} - - {m.runResultsNoteLink()} - - . -

      -
      -
    6. -
    -
    -
    - - {/* Agents & models */} -
    -
    -

    - ✦ {m.runAgentsEyebrow()} -

    -

    - {m.runAgentsTitle()} -

    -

    - {m.runAgentsDesc()} -

    - -
    - - - - - - - - - - {AGENTS.map((a) => ( - - - - - - ))} - -
    {m.runColAgent()}{m.runColModels()}{m.runColAuth()}
    {a.agent} - {a.models} - {a.auth}
    -
    -
    -
    - - {/* Flags */} -
    -
    -

    - ✦ {m.runFlagsEyebrow()} -

    -

    - {m.runFlagsTitle()} -

    -
      - {FLAGS.map((f) => ( -
    • - - {f.flag} - - {flagDesc(f.id)} -
    • - ))} -
    -
    -
    - - {/* CTA */} -
    -

    - ✦ {m.runCtaEyebrow()} -

    -

    - {m.runCtaTitle()} -

    -

    - {m.runCtaDesc()} -

    - -
    - -
    -
    -
    - ); -} - -function StepLabel({ n, title }: { n: number; title: string }) { - return ( -
    - - {n} - - {title} -
    - ); -} diff --git a/apps/web/test/content/seo-contract.test.ts b/apps/web/test/content/seo-contract.test.ts index 10e637c3a..4e4f5160e 100644 --- a/apps/web/test/content/seo-contract.test.ts +++ b/apps/web/test/content/seo-contract.test.ts @@ -52,7 +52,7 @@ describe("SEO contracts", () => { }); }); - it("includes docs, guides, stack pages, MCP, and the benchmark runner in the dynamic sitemap", () => { + it("includes docs, guides, stack pages, MCP, and the benchmark page in the dynamic sitemap", () => { const entries = getSitemapEntriesFromPages({ docsPages: [ { slug: [], frontmatter: { updated: "2026-05-12" } }, @@ -78,7 +78,7 @@ describe("SEO contracts", () => { expect(paths).toContain("/docs/cli/create"); expect(paths).toContain("/guides/typescript/create-tanstack-start-project"); expect(paths).toContain("/mcp"); - expect(paths).toContain("/run"); + expect(paths).toContain("/benchmark"); expect(paths).toContain("/templates"); expect(paths).not.toContain("/stack"); expect(paths).toContain("/stack/nextjs-hono-drizzle-better-auth"); diff --git a/apps/web/test/stack/verified-combinations-badge.test.ts b/apps/web/test/stack/verified-combinations-badge.test.ts index 8b6ed4b7b..c2a1162f1 100644 --- a/apps/web/test/stack/verified-combinations-badge.test.ts +++ b/apps/web/test/stack/verified-combinations-badge.test.ts @@ -114,7 +114,7 @@ describe("public verification receipt and badge", () => { }); expect(badge).toMatchObject({ color: "brightgreen", message: "8/8 runtime verified" }); expect(verification.cases[1]?.runtimeLimitation).toContain("native device UI"); - expect(JSON.stringify(badge).toLowerCase()).not.toContain("scaffbench"); + expect(JSON.stringify(badge).toLowerCase()).not.toContain("fixproof"); }); it("fails closed when the receipt is missing or malformed", () => { diff --git a/apps/web/vercel.json b/apps/web/vercel.json index 25efcd027..942f2c11b 100644 --- a/apps/web/vercel.json +++ b/apps/web/vercel.json @@ -56,6 +56,11 @@ "destination": "/benchmark", "permanent": true }, + { + "source": "/run", + "destination": "/benchmark", + "permanent": true + }, { "source": "/docs/recipes/default-typescript-web.md", "destination": "/guides/typescript/hono-trpc-drizzle.md", diff --git a/benchmarks/README.md b/benchmarks/README.md deleted file mode 100644 index 0b9828bbe..000000000 --- a/benchmarks/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# ScaffBench reports - -ScaffBench measures how well AI coding agents **start real full-stack projects**. Not edit existing -code, but scaffold a working project from scratch. The core question: does the generated project -actually install and build? - -This directory holds the open-sourced run reports for the current suite. Reports from suites 1, 2, -and 2.1 were removed when suite 3.0 reset the protocol. Their numbers are not comparable to 3.0 -results, so keeping them here invited apples-to-oranges reading. Recover them from git history if -you need them. - -``` -benchmarks/ - -/ one folder per published run of suite 3.0 -``` - -Each run folder contains: - -- **`summary.json`**: the aggregate leaderboard, per-spec cells (`bySpecCell`), and per-run results - (validation steps, wired-libraries score, cost, tokens). -- **`summary.md`**: a human-readable version of the same. - -## What's not here - -The raw generated projects and their build artifacts (`node_modules`, cargo/target, .venv) run to -roughly 16 GB and stay out of the repo. These reports are the scored summaries. The numbers, not the -gigabytes. - -## How runs are scored - -- **Core pass@1**: does the project install, build, type-check, and native-compile (`cargo check`, - `go build`, `dotnet build`, `mvn`, `mix`)? Everything hinges on this. -- **Quality pass@1**: Core plus lint and format green on a clean machine. Tests run and are - reported, but they do not affect any score. -- **Wired libraries**: did the agent actually use the libraries the spec calls for? Scored against - the dependencies, imports, and files present in the generated tree, not names it mentioned. - Trap and restraint markers (a forbidden ORM, a forbidden build tool) live in the same score. -- **Index** (0-100): the headline and the board's sort key. Every spec earns a graded score, 0.6 for a Core pass, - 0.2 for the share of lint and format gates green, and 0.2 for wired libraries. Tests are not - scored, since the harness can only run the tests the model wrote. The index is the mean of those - scores weighted by spec difficulty (1 easy, 2 hard, 3 frontier, pinned in each spec), times 100. - Cost, time, and lines of code are shown beside the index and never enter it. -- **Run outcome**: every run is `success`, `model-failure`, or `infra-inconclusive`. Toolchain stalls - and un-measurable runs are excluded from the rate. A generation timeout counts as a model failure, - as in SWE-bench. - -Suite 3.0 scores the prompt path only: no scaffolder, the agent hand-writes every file. That is the -purest measure of raw capability. - -The live board is at [better-fullstack.com/benchmark](https://better-fullstack.com/benchmark). diff --git a/benchmarks/gemini-3-7-flash-low/summary.json b/benchmarks/gemini-3-7-flash-low/summary.json deleted file mode 100644 index 928c8a986..000000000 --- a/benchmarks/gemini-3-7-flash-low/summary.json +++ /dev/null @@ -1,6260 +0,0 @@ -{ - "harnessVersion": "3.1.0", - "generatedAt": "2026-08-27T17:58:37.987Z", - "options": { - "command": "run", - "model": "gemini-3.7-flash", - "efforts": [ - "low" - ], - "paths": [ - "prompt" - ], - "specs": [ - "ai-search-workbench", - "rust-leptos-axum", - "python-ingestion-api", - "go-realtime-api", - "multi-dotnet-ops", - "ts-svelte-edge-orpc", - "dotnet-blazor-cqrs", - "multi-ts-go-grpc", - "java-spring-jooq-keycloak", - "elixir-broadway-absinthe", - "react-native-expo", - "frontier-polyglot-proto", - "frontier-effect-eventsourcing" - ], - "repeats": 1, - "outDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27", - "maxBudgetUsd": "12", - "skipValidation": false, - "generateOnly": false, - "validateExisting": false, - "forceRevalidate": false, - "qualityGate": true, - "noQualityGate": false, - "doctorCheck": false, - "routeCheck": false, - "promptStyle": "explicit", - "repair": false - }, - "metadata": { - "cwd": "/home/ibrahim/code/Better-Fullstack", - "evidenceSchemaVersion": 1, - "workspaceClean": false, - "gitHead": "38ccd8d1d2dcedf7bb5738716af96f7d3142e618", - "gitBranch": "feat/scaffbench-3-luna-board", - "bunVersion": "1.4.0", - "nodeVersion": "v26.3.0", - "platform": "linux", - "arch": "x64", - "environmentQualified": true, - "toolchains": { - "bun": "1.4.0", - "node": "v24.19.0", - "rustc": "rustc 1.98.0 (88d9e12ae 2026-08-18)", - "cargo": "cargo 1.98.0 (797e8a9bc 2026-08-05)", - "go": "go version go1.27.0 linux/amd64", - "dotnet": "10.0.400", - "python": "Python 3.12.3", - "uv": "uv 0.12.5 (x86_64-unknown-linux-gnu)", - "java": "openjdk 21.0.12.1 2026-08-18 LTS\nOpenJDK Runtime Environment Temurin-21.0.12.1+1 (build 21.0.12.1+1-LTS)\nOpenJDK 64-Bit Server VM Temurin-21.0.12.1+1 (build 21.0.12.1+1-LTS, mixed mode, sharing)", - "mvn": "Apache Maven 3.9.16 (2bdd9fddda4b155ebf8000e807eb73fd829a51d5)\nMaven home: /home/ibrahim/.local/share/mise/installs/maven/latest/apache-maven-3.9.16\nJava version: 21.0.12.1, vendor: Eclipse Adoptium, runtime: /home/ibrahim/.local/share/mise/installs/java/temurin-21.0.12+101.0.LTS\nDefault locale: en_US, platform encoding: UTF-8\nOS name: \"linux\", version: \"7.0.0-30-generic\", arch: \"amd64\", family: \"unix\"", - "mix": "Erlang/OTP 29 [erts-17.0.5] [source] [64-bit] [smp:6:6] [ds:6:6:10] [async-threads:1] [jit:ns]\n\nMix 1.20.3 (compiled with Erlang/OTP 29)", - "buf": "1.72.0", - "protoc": "libprotoc 35.1", - "psql": "psql (PostgreSQL) 16.15 (Ubuntu 16.15-0ubuntu0.24.04.1)" - }, - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "agentAdapter": "agy", - "configuredTrials": 1, - "bfGeneratorVersion": "2.6.2", - "model": "gemini-3.7-flash", - "effectiveReasoning": [ - { - "effort": "low", - "effectiveReasoning": "low" - } - ], - "specOrderSeed": 847190081, - "runProtocol": { - "repeats": 1, - "seed": 847190081 - }, - "publicationEligibility": { - "gemini-3.7-flash|low|prompt": "ranked" - } - }, - "specs": [ - { - "id": "ai-search-workbench", - "introducedAt": "2026-08-21", - "title": "AI search workbench with split semantic/full-text search on the Vite+ toolchain", - "lane": "core", - "difficulty": 2, - "family": "typescript", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a TypeScript monorepo for an AI support/search workbench: agents triage tickets, search past resolutions semantically, and admins search raw documents.", - "Use a TanStack Router web app styled with Tailwind and shadcn/ui.", - "Use a Hono backend on Bun.", - "Use oRPC for the app API.", - "Use PostgreSQL with Drizzle for relational app data only.", - "Use Better Auth for accounts.", - "Use the Vercel AI SDK for AI features.", - "Use Qdrant for semantic vector search over embeddings. Postgres is present, but pgvector is not acceptable here, the vector workload must live in Qdrant.", - "Use OpenSearch for full-text document/admin search. Do not collapse the two search modes into one engine.", - "Use Inngest for background indexing jobs.", - "Use Pino logging and OpenTelemetry instrumentation.", - "Use TanStack Store, TanStack Form, Valibot, Vitest + Playwright, and Paraglide.", - "Use the Vite+ toolchain (`vp`) for workspace scripts, linting, and formatting. Do not add Turborepo, Nx, or Biome.", - "Include DevContainer and GitHub Actions CI output.", - "Do not add payments, email, realtime, CMS, file upload, file storage, feature flags, or deploy targets." - ], - "naturalPrompt": "Build a production-grade AI support search starter. Support agents need semantic search over embedded past resolutions, admins need full-text search over raw documents, and the two workloads should not share one engine. It also needs account auth, relational app data, background indexing, observability, tests, i18n, CI, and a workspace toolchain for tasks, lint, and format. Keep it deploy-target neutral and do not include commerce/email/storage extras.", - "rightLibraryNotes": [ - "Qdrant must be used for semantic vector search; pgvector is not acceptable even though Postgres is present.", - "OpenSearch must be used for full-text document/admin search.", - "Inngest must be used for background indexing jobs.", - "oRPC must be used for the app API.", - "The Vite+ toolchain (vp) replaces Turborepo/Nx and Biome for workspace tasks, lint, and format." - ], - "canonicalFlags": [ - "--ecosystem", - "typescript", - "--frontend", - "tanstack-router", - "--backend", - "hono", - "--runtime", - "bun", - "--api", - "orpc", - "--database", - "postgres", - "--orm", - "drizzle", - "--db-setup", - "none", - "--auth", - "better-auth", - "--payments", - "none", - "--email", - "none", - "--file-upload", - "none", - "--logging", - "pino", - "--observability", - "opentelemetry", - "--feature-flags", - "none", - "--analytics", - "none", - "--effect", - "none", - "--state-management", - "tanstack-store", - "--forms", - "tanstack-form", - "--validation", - "valibot", - "--testing", - "vitest-playwright", - "--ai", - "vercel-ai", - "--realtime", - "none", - "--job-queue", - "inngest", - "--animation", - "none", - "--css-framework", - "tailwind", - "--ui-library", - "shadcn-ui", - "--cms", - "none", - "--caching", - "none", - "--rate-limit", - "none", - "--i18n", - "paraglide", - "--search", - "opensearch", - "--vector-db", - "qdrant", - "--file-storage", - "none", - "--web-deploy", - "none", - "--server-deploy", - "none", - "--addons", - "vite-plus", - "devcontainer", - "github-actions", - "--examples", - "none", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--shadcn-base", - "radix", - "--shadcn-style", - "nova", - "--shadcn-icon-library", - "lucide", - "--shadcn-color-theme", - "neutral", - "--shadcn-base-color", - "neutral", - "--shadcn-font", - "inter", - "--shadcn-radius", - "default", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "frontend": "tanstack-router", - "backend": "hono", - "runtime": "bun", - "api": "orpc", - "database": "postgres", - "orm": "drizzle", - "auth": "better-auth", - "ai": "vercel-ai", - "vectorDb": "qdrant", - "search": "opensearch", - "jobQueue": "inngest", - "logging": "pino", - "observability": "opentelemetry", - "stateManagement": "tanstack-store", - "forms": "tanstack-form", - "validation": "valibot", - "testing": "vitest-playwright", - "i18n": "paraglide", - "cssFramework": "tailwind", - "uiLibrary": "shadcn-ui" - }, - "expectedAddons": [ - "vite-plus", - "devcontainer", - "github-actions" - ], - "strictMarkers": [ - { - "id": "frontend:tanstack-router", - "deps": [ - "@tanstack/react-router" - ] - }, - { - "id": "css:tailwind", - "deps": [ - "tailwindcss" - ] - }, - { - "id": "ui:shadcn", - "files": [ - "components.json" - ] - }, - { - "id": "backend:hono", - "deps": [ - "hono" - ] - }, - { - "id": "api:orpc", - "deps": [ - "@orpc/server" - ], - "source": [ - "@orpc/" - ] - }, - { - "id": "database:postgres+drizzle", - "deps": [ - "drizzle-orm" - ], - "source": [ - "drizzle-orm" - ] - }, - { - "id": "auth:better-auth", - "deps": [ - "better-auth" - ], - "source": [ - "better-auth" - ] - }, - { - "id": "ai:vercel-ai", - "deps": [ - "ai" - ] - }, - { - "id": "vectorDb:qdrant", - "deps": [ - "@qdrant/js-client-rest" - ], - "source": [ - "@qdrant/js-client-rest" - ] - }, - { - "id": "search:opensearch", - "deps": [ - "@opensearch-project/opensearch" - ], - "source": [ - "@opensearch-project/opensearch" - ] - }, - { - "id": "jobQueue:inngest", - "deps": [ - "inngest" - ], - "source": [ - "inngest" - ] - }, - { - "id": "logging:pino", - "deps": [ - "pino" - ], - "source": [ - "pino" - ] - }, - { - "id": "observability:opentelemetry", - "deps": [ - "@opentelemetry/api" - ] - }, - { - "id": "state:tanstack-store", - "deps": [ - "@tanstack/store" - ] - }, - { - "id": "forms:tanstack-form", - "deps": [ - "@tanstack/react-form" - ] - }, - { - "id": "validation:valibot", - "deps": [ - "valibot" - ] - }, - { - "id": "testing:vitest-playwright", - "deps": [ - "vitest", - "@playwright/test" - ] - }, - { - "id": "i18n:paraglide", - "deps": [ - "@inlang/paraglide-js" - ], - "files": [ - "project.inlang/settings.json" - ] - }, - { - "id": "addon:vite-plus", - "deps": [ - "vite-plus" - ] - }, - { - "id": "addon:devcontainer", - "files": [ - ".devcontainer/devcontainer.json" - ] - }, - { - "id": "addon:github-actions", - "files": [ - ".github/workflows/*.y*ml" - ] - }, - { - "id": "forbidden:pgvector", - "forbiddenDeps": [ - "pgvector" - ] - }, - { - "id": "forbidden:turborepo", - "forbiddenDeps": [ - "turbo" - ] - }, - { - "id": "forbidden:nx", - "forbiddenDeps": [ - "nx", - "@nx/" - ], - "forbiddenFiles": [ - "nx.json" - ] - }, - { - "id": "forbidden:biome", - "forbiddenDeps": [ - "@biomejs/biome" - ] - }, - { - "id": "forbidden:payments", - "forbiddenDeps": [ - "stripe", - "@stripe/stripe-js", - "polar-sh" - ] - }, - { - "id": "forbidden:email", - "forbiddenDeps": [ - "resend", - "nodemailer", - "@react-email/components" - ] - } - ], - "acceptanceSets": { - "web-framework": [ - "@tanstack/react-router", - "next", - "react-router", - "@remix-run", - "vite" - ], - "backend": [ - "hono", - "express", - "fastify", - "elysia", - "@nestjs/core" - ], - "relational-db": [ - "drizzle-orm", - "@prisma/client", - "kysely", - "typeorm", - "sequelize" - ], - "auth": [ - "better-auth", - "lucia", - "@clerk", - "next-auth", - "@auth/", - "@workos-inc" - ], - "ai": [ - "ai", - "@ai-sdk", - "openai", - "@anthropic-ai", - "langchain" - ], - "semantic-search": [ - "@qdrant/js-client-rest", - "pgvector", - "weaviate-ts-client", - "@pinecone-database/pinecone", - "chromadb", - "@zilliz/milvus2-sdk-node" - ], - "full-text-search": [ - "@opensearch-project/opensearch", - "@elastic/elasticsearch", - "meilisearch", - "typesense", - "algoliasearch" - ], - "background-jobs": [ - "inngest", - "bullmq", - "@trigger.dev", - "graphile-worker", - "pg-boss", - "bee-queue" - ], - "observability": [ - "@opentelemetry/api", - "pino", - "winston", - "@sentry", - "@logtail" - ], - "testing": [ - "vitest", - "jest", - "@playwright/test", - "cypress" - ], - "i18n": [ - "@inlang/paraglide-js", - "next-intl", - "i18next", - "react-i18next", - "lingui" - ], - "toolchain": [ - "vite-plus", - "turbo", - "nx" - ], - "ci": [ - ".github/workflows", - ".gitlab-ci.yml", - ".circleci" - ] - }, - "validationProfile": { - "packageManager": "bun", - "qualityGate": true, - "doctorCheck": true, - "routeCheckCandidate": true - } - }, - { - "id": "rust-leptos-axum", - "introducedAt": "2026-08-21", - "title": "Rust Axum API with a Leptos WASM frontend and typed service libraries", - "lane": "core", - "difficulty": 1, - "family": "rust", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a Rust project for an internal feature-flag console with Axum as the backend web framework (not Actix).", - "Use Leptos specifically for the WASM frontend; do not replace it with Dioxus, Yew, or a JavaScript frontend.", - "Use PostgreSQL with SQLx.", - "Use Tonic for a typed API boundary.", - "Include Clap CLI support, tracing, anyhow/thiserror, Moka caching, OAuth2 auth, Lapin jobs, OpenTelemetry, and Askama templates.", - "Include serde, uuid, chrono, reqwest, config, utoipa, validator, and tokio-test libraries." - ], - "naturalPrompt": "Build a Rust starter for an internal feature-flag console. It should have an Axum server, a Rust WASM frontend, Postgres access, typed service/API boundaries, CLI/admin utilities, tracing, auth, cache, jobs, and template rendering. Choose the right Rust libraries rather than swapping in web defaults.", - "rightLibraryNotes": [ - "Leptos is required for the Rust WASM frontend.", - "Axum is required for the server.", - "SQLx is required for database access.", - "Tonic is required for typed RPC/API contracts." - ], - "canonicalFlags": [ - "--ecosystem", - "rust", - "--database", - "postgres", - "--rust-web-framework", - "axum", - "--rust-frontend", - "leptos", - "--rust-orm", - "sqlx", - "--rust-api", - "tonic", - "--rust-cli", - "clap", - "--rust-libraries", - "serde", - "uuid", - "chrono", - "reqwest", - "config", - "utoipa", - "validator", - "tokio-test", - "--rust-logging", - "tracing", - "--rust-error-handling", - "anyhow-thiserror", - "--rust-caching", - "moka", - "--rust-auth", - "oauth2", - "--rust-realtime", - "none", - "--rust-message-queue", - "lapin", - "--rust-observability", - "opentelemetry", - "--rust-templating", - "askama", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "none", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "rust", - "database": "postgres", - "rustWebFramework": "axum", - "rustFrontend": "leptos", - "rustOrm": "sqlx", - "rustApi": "tonic", - "rustCli": "clap", - "rustLibraries": [ - "serde", - "uuid", - "chrono", - "reqwest", - "config", - "utoipa", - "validator", - "tokio-test" - ], - "rustLogging": "tracing", - "rustErrorHandling": "anyhow-thiserror", - "rustCaching": "moka", - "rustAuth": "oauth2", - "rustMessageQueue": "lapin", - "rustObservability": "opentelemetry", - "rustTemplating": "askama" - }, - "strictMarkers": [ - { - "id": "rust:axum", - "text": [ - "axum" - ] - }, - { - "id": "frontend:leptos", - "text": [ - "leptos", - "leptos_router" - ], - "files": [ - "*/Cargo.toml" - ] - }, - { - "id": "orm:sqlx", - "text": [ - "sqlx" - ] - }, - { - "id": "db:postgres", - "textAny": [ - "PgPool", - "sqlx::postgres", - "sqlx::Postgres", - "tokio-postgres" - ] - }, - { - "id": "api:tonic", - "text": [ - "tonic" - ] - }, - { - "id": "cli:clap", - "text": [ - "clap" - ] - }, - { - "id": "logging:tracing", - "text": [ - "tracing" - ] - }, - { - "id": "cache:moka", - "text": [ - "moka" - ] - }, - { - "id": "auth:oauth2", - "text": [ - "oauth2" - ] - }, - { - "id": "jobs:lapin", - "text": [ - "lapin" - ] - }, - { - "id": "observability:opentelemetry", - "text": [ - "opentelemetry" - ] - }, - { - "id": "templating:askama", - "text": [ - "askama" - ] - }, - { - "id": "errors:anyhow-thiserror", - "text": [ - "anyhow", - "thiserror" - ] - }, - { - "id": "lib:serde", - "text": [ - "serde" - ] - }, - { - "id": "lib:uuid", - "text": [ - "uuid" - ] - }, - { - "id": "lib:chrono", - "text": [ - "chrono" - ] - }, - { - "id": "lib:reqwest", - "text": [ - "reqwest" - ] - }, - { - "id": "lib:config", - "textAny": [ - "config = \"", - "config = {", - "config.workspace" - ] - }, - { - "id": "lib:utoipa", - "text": [ - "utoipa" - ] - }, - { - "id": "lib:validator", - "text": [ - "validator" - ] - }, - { - "id": "lib:tokio-test", - "text": [ - "tokio-test" - ] - }, - { - "id": "forbidden:dioxus", - "forbiddenText": [ - "dioxus-router", - "dioxus::prelude", - "dioxus = {", - "dioxus.workspace" - ], - "forbiddenFiles": [ - "Dioxus.toml" - ] - }, - { - "id": "forbidden:actix", - "forbiddenText": [ - "actix-web" - ] - }, - { - "id": "forbidden:yew", - "forbiddenText": [ - "yew::prelude", - "yew = {", - "yew = \"", - "yew.workspace" - ] - } - ], - "validationProfile": { - "native": [ - "cargo" - ] - } - }, - { - "id": "python-ingestion-api", - "introducedAt": "2026-08-21", - "title": "Python FastAPI ingestion API with AI, queues, realtime, and quality gates", - "lane": "core", - "difficulty": 1, - "family": "python", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a Python API project using FastAPI for an AI document-ingestion pipeline: upload, queue, extract, and stream progress.", - "Use SQLModel for database models and Pydantic for validation.", - "Use LangGraph and OpenAI SDK for the extraction workflow.", - "Use JWT auth, Celery task queues, WebSockets for realtime job progress, Redis caching, OpenTelemetry, Typer, Rich, Ruff, Pytest, and Hypothesis.", - "Do not choose Django REST Framework, Django Ninja, or Flask, this is a FastAPI project." - ], - "naturalPrompt": "Build a Python ingestion API starter for AI document processing. It needs FastAPI, SQL-backed models, strict validation, AI workflow libraries, queued workers, realtime job updates, Redis cache, tracing, CLI tools, and real test/quality tooling. Avoid Django-only API libraries.", - "rightLibraryNotes": [ - "FastAPI is required; Django-specific API packages are forbidden.", - "SQLModel is required for the database layer.", - "LangGraph plus OpenAI SDK are required for AI workflow scaffolding.", - "Celery is required for background ingestion jobs." - ], - "canonicalFlags": [ - "--ecosystem", - "python", - "--database", - "postgres", - "--python-web-framework", - "fastapi", - "--python-orm", - "sqlmodel", - "--python-validation", - "pydantic", - "--python-ai", - "langgraph", - "openai-sdk", - "--python-auth", - "jwt", - "--python-api", - "none", - "--python-task-queue", - "celery", - "--python-graphql", - "none", - "--python-quality", - "ruff", - "--python-testing", - "pytest", - "hypothesis", - "--python-caching", - "redis", - "--python-realtime", - "websockets", - "--python-observability", - "opentelemetry", - "--python-cli", - "typer", - "rich", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "none", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "python", - "database": "postgres", - "pythonWebFramework": "fastapi", - "pythonOrm": "sqlmodel", - "pythonValidation": "pydantic", - "pythonAi": [ - "langgraph", - "openai-sdk" - ], - "pythonAuth": "jwt", - "pythonApi": "none", - "pythonTaskQueue": "celery", - "pythonQuality": "ruff", - "pythonTesting": [ - "pytest", - "hypothesis" - ], - "pythonCaching": "redis", - "pythonRealtime": "websockets", - "pythonObservability": "opentelemetry", - "pythonCli": [ - "typer", - "rich" - ] - }, - "strictMarkers": [ - { - "id": "backend:fastapi", - "text": [ - "fastapi" - ] - }, - { - "id": "orm:sqlmodel", - "text": [ - "sqlmodel" - ] - }, - { - "id": "validation:pydantic", - "text": [ - "pydantic" - ] - }, - { - "id": "ai:langgraph", - "text": [ - "langgraph" - ] - }, - { - "id": "ai:openai-sdk", - "text": [ - "openai" - ] - }, - { - "id": "auth:jwt", - "text": [ - "jwt" - ] - }, - { - "id": "jobs:celery", - "text": [ - "celery" - ] - }, - { - "id": "quality:ruff", - "text": [ - "ruff" - ] - }, - { - "id": "testing:pytest", - "text": [ - "pytest" - ] - }, - { - "id": "testing:hypothesis", - "text": [ - "hypothesis" - ] - }, - { - "id": "realtime:websockets", - "text": [ - "websockets" - ] - }, - { - "id": "cli:typer+rich", - "text": [ - "typer", - "rich" - ] - }, - { - "id": "caching:redis", - "text": [ - "redis" - ] - }, - { - "id": "observability:opentelemetry", - "textAny": [ - "opentelemetry" - ] - }, - { - "id": "forbidden:django-api", - "forbiddenText": [ - "djangorestframework", - "django-rest-framework", - "rest_framework", - "django-ninja", - "django_ninja" - ] - }, - { - "id": "forbidden:flask", - "forbiddenText": [ - "flask" - ] - } - ], - "validationProfile": { - "native": [ - "python" - ] - } - }, - { - "id": "go-realtime-api", - "introducedAt": "2026-08-21", - "title": "Go realtime API with Chi, Ent, gRPC, NATS, Redis, and OpenTelemetry", - "lane": "core", - "difficulty": 1, - "family": "go", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a Go API project for a fleet-tracking admin service: vehicles report positions, operators watch them live.", - "Use Chi as the router, not Gin/Echo/Fiber.", - "Use PostgreSQL with Ent.", - "Use gRPC for the vehicle-ingest service contract.", - "Use Cobra CLI tooling, Zap logging, JWT auth, Testify + GoMock tests, Gorilla WebSocket for live position updates, NATS messaging, Redis caching, Viper config, and OpenTelemetry." - ], - "naturalPrompt": "Build a Go backend starter for a realtime admin API. It needs a lightweight router, Ent/Postgres models, typed gRPC service contracts, CLI/admin commands, structured logging, auth, websocket updates, event messaging, Redis cache, configuration, tracing, and test doubles.", - "rightLibraryNotes": [ - "Chi is required as the web framework.", - "Ent is required for the data layer.", - "gRPC-Go is required for typed service contracts.", - "NATS and Gorilla WebSocket are required for messaging and realtime updates." - ], - "canonicalFlags": [ - "--ecosystem", - "go", - "--database", - "postgres", - "--go-web-framework", - "chi", - "--go-orm", - "ent", - "--go-api", - "grpc-go", - "--go-cli", - "cobra", - "--go-logging", - "zap", - "--go-auth", - "jwt", - "--go-testing", - "testify", - "gomock", - "--go-realtime", - "gorilla-websocket", - "--go-message-queue", - "nats", - "--go-caching", - "redis", - "--go-config", - "viper", - "--go-observability", - "opentelemetry", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "none", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "go", - "database": "postgres", - "goWebFramework": "chi", - "goOrm": "ent", - "goApi": "grpc-go", - "goCli": "cobra", - "goLogging": "zap", - "goAuth": "jwt", - "goTesting": [ - "testify", - "gomock" - ], - "goRealtime": "gorilla-websocket", - "goMessageQueue": "nats", - "goCaching": "redis", - "goConfig": "viper", - "goObservability": "opentelemetry" - }, - "strictMarkers": [ - { - "id": "backend:chi", - "text": [ - "github.com/go-chi/chi" - ] - }, - { - "id": "orm:ent", - "text": [ - "entgo.io/ent" - ], - "files": [ - "ent/schema/*.go" - ] - }, - { - "id": "db:postgres", - "textAny": [ - "github.com/lib/pq", - "github.com/jackc/pgx" - ] - }, - { - "id": "api:grpc-go", - "text": [ - "google.golang.org/grpc" - ] - }, - { - "id": "cli:cobra", - "text": [ - "github.com/spf13/cobra" - ] - }, - { - "id": "logging:zap", - "text": [ - "go.uber.org/zap" - ] - }, - { - "id": "auth:jwt", - "text": [ - "github.com/golang-jwt/jwt" - ] - }, - { - "id": "testing:testify+gomock", - "text": [ - "github.com/stretchr/testify", - "go.uber.org/mock" - ] - }, - { - "id": "realtime:gorilla-websocket", - "text": [ - "github.com/gorilla/websocket" - ] - }, - { - "id": "queue:nats", - "text": [ - "github.com/nats-io/nats.go" - ] - }, - { - "id": "caching:redis", - "text": [ - "github.com/redis/go-redis" - ] - }, - { - "id": "config:viper", - "text": [ - "github.com/spf13/viper" - ] - }, - { - "id": "observability:opentelemetry", - "text": [ - "go.opentelemetry.io/otel" - ] - }, - { - "id": "forbidden:gin", - "forbiddenText": [ - "github.com/gin-gonic/gin" - ] - }, - { - "id": "forbidden:echo", - "forbiddenText": [ - "github.com/labstack/echo" - ] - }, - { - "id": "forbidden:fiber", - "forbiddenText": [ - "github.com/gofiber/fiber" - ] - } - ], - "validationProfile": { - "native": [ - "go" - ] - } - }, - { - "id": "multi-dotnet-ops", - "introducedAt": "2026-08-21", - "title": "Multi-ecosystem ops portal with TypeScript frontend and .NET Minimal API backend", - "lane": "core", - "difficulty": 2, - "family": "multi-ecosystem", - "supportedByBetterFullstack": true, - "requirements": [ - "Create one multi-ecosystem project graph for an incident-ops portal: on-call engineers acknowledge incidents in the web UI, the backend fans out notifications.", - "Use a Next.js TypeScript frontend with Tailwind and shadcn/ui.", - "Use an ASP.NET Minimal API backend.", - "Use EF Core, ASP.NET Identity, Minimal API endpoints, xUnit, Testcontainers for .NET, Serilog, SignalR for live incident updates, FluentValidation, Hangfire for notification fan-out, memory cache, and Docker output.", - "Use PostgreSQL as the shared database.", - "Include Turborepo, Biome, and GitHub Actions." - ], - "naturalPrompt": "Build a multi-ecosystem ops portal starter: a TypeScript web frontend and a .NET backend. It needs Postgres-backed identity, API endpoints, validation, background jobs, realtime notifications, observability/logging, tests, containers, and CI. Use the project graph instead of forcing everything into one ecosystem.", - "rightLibraryNotes": [ - "The frontend must be TypeScript Next.js.", - "The backend must be ASP.NET Minimal API.", - "EF Core and ASP.NET Identity are required.", - "Hangfire and SignalR are required for jobs and realtime updates." - ], - "canonicalFlags": [ - "--part", - "frontend:typescript:next", - "--part", - "frontend.css:typescript:tailwind", - "--part", - "frontend.ui:typescript:shadcn-ui", - "--part", - "backend:dotnet:aspnet-minimal", - "--part", - "backend.orm:dotnet:ef-core", - "--part", - "backend.auth:dotnet:aspnet-identity", - "--part", - "backend.api:dotnet:minimal-api", - "--part", - "backend.testing:dotnet:xunit", - "--part", - "backend.testing:dotnet:testcontainers-dotnet", - "--part", - "backend.observability:dotnet:serilog", - "--part", - "backend.realtime:dotnet:signalr", - "--part", - "backend.validation:dotnet:fluentvalidation", - "--part", - "backend.jobQueue:dotnet:hangfire", - "--part", - "backend.caching:dotnet:memory-cache", - "--part", - "backend.deploy:dotnet:docker", - "--part", - "database:universal:postgres", - "--addons", - "turborepo", - "biome", - "github-actions", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--shadcn-base", - "radix", - "--shadcn-style", - "nova", - "--shadcn-icon-library", - "lucide", - "--shadcn-color-theme", - "neutral", - "--shadcn-base-color", - "neutral", - "--shadcn-font", - "inter", - "--shadcn-radius", - "default", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedParts": [ - "frontend:typescript:next", - "frontend.css:typescript:tailwind", - "frontend.ui:typescript:shadcn-ui", - "backend:dotnet:aspnet-minimal", - "backend.orm:dotnet:ef-core", - "backend.auth:dotnet:aspnet-identity", - "backend.api:dotnet:minimal-api", - "backend.testing:dotnet:xunit", - "backend.testing:dotnet:testcontainers-dotnet", - "backend.observability:dotnet:serilog", - "backend.realtime:dotnet:signalr", - "backend.validation:dotnet:fluentvalidation", - "backend.jobQueue:dotnet:hangfire", - "backend.caching:dotnet:memory-cache", - "backend.deploy:dotnet:docker", - "database:universal:postgres" - ], - "expectedAddons": [ - "turborepo", - "biome", - "github-actions" - ], - "strictMarkers": [ - { - "id": "frontend:next", - "deps": [ - "next" - ] - }, - { - "id": "frontend:tailwind", - "deps": [ - "tailwindcss" - ] - }, - { - "id": "frontend:shadcn", - "files": [ - "components.json" - ] - }, - { - "id": "backend:aspnet-minimal", - "files": [ - "Program.cs" - ], - "text": [ - "MapGet" - ] - }, - { - "id": "orm:ef-core", - "text": [ - "Microsoft.EntityFrameworkCore" - ] - }, - { - "id": "db:postgres", - "textAny": [ - "Npgsql" - ] - }, - { - "id": "auth:aspnet-identity", - "text": [ - "Microsoft.AspNetCore.Identity" - ] - }, - { - "id": "testing:xunit", - "text": [ - "xunit" - ] - }, - { - "id": "testing:testcontainers", - "text": [ - "Testcontainers" - ] - }, - { - "id": "logging:serilog", - "text": [ - "Serilog" - ] - }, - { - "id": "realtime:signalr", - "text": [ - "SignalR" - ] - }, - { - "id": "validation:fluentvalidation", - "text": [ - "FluentValidation" - ] - }, - { - "id": "jobs:hangfire", - "text": [ - "Hangfire" - ] - }, - { - "id": "caching:memory-cache", - "textAny": [ - "IMemoryCache", - "AddMemoryCache", - "Microsoft.Extensions.Caching.Memory" - ] - }, - { - "id": "deploy:docker", - "files": [ - "Dockerfile" - ] - }, - { - "id": "addon:turborepo", - "deps": [ - "turbo" - ] - }, - { - "id": "addon:biome", - "deps": [ - "@biomejs/biome" - ] - }, - { - "id": "addon:github-actions", - "files": [ - ".github/workflows/*.y*ml" - ] - } - ], - "validationProfile": { - "packageManager": "bun", - "native": [ - "dotnet" - ], - "qualityGate": true, - "doctorCheck": true - } - }, - { - "id": "ts-svelte-edge-orpc", - "introducedAt": "2026-08-21", - "title": "SvelteKit edge app on Cloudflare Workers with Hono + oRPC and D1", - "lane": "core", - "difficulty": 2, - "family": "typescript", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a TypeScript monorepo for an edge-deployed link-in-bio app: public pages render at the edge, signed-in users edit their page.", - "Use a SvelteKit web frontend styled with Tailwind.", - "Use a Hono backend running on the Cloudflare Workers runtime.", - "Use oRPC for the app API. tRPC is the obvious pick and the wrong one, it does not support a Svelte frontend.", - "Use SQLite via Cloudflare D1 with Drizzle as the ORM.", - "Use Better Auth for accounts and Valibot for validation. Every library must actually run on Workers, no Node-only APIs.", - "Deploy both the web app and the server to Cloudflare. The SvelteKit build must use the Cloudflare adapter; a Node server adapter is a failure.", - "Do not add payments, email, realtime, search, vector DB, jobs, CMS, file storage/upload, analytics, or i18n." - ], - "naturalPrompt": "Build an edge-first starter that runs on Cloudflare. It needs a Svelte web app, a lightweight server on the Workers runtime, a type-safe app API, an edge SQL database with a typed ORM, account auth, and validation. Every choice has to actually run on Workers, resolve the conflicts that creates instead of reaching for Node defaults, and both apps deploy to Cloudflare.", - "rightLibraryNotes": [ - "oRPC is required for the API because tRPC does not support a Svelte frontend.", - "The Workers runtime requires the Hono backend.", - "Cloudflare D1 is required for the SQLite database, and Cloudflare for deploys.", - "Better Auth must use a Workers-compatible ORM (Drizzle).", - "The SvelteKit build must target the Cloudflare adapter, not a Node server adapter." - ], - "canonicalFlags": [ - "--ecosystem", - "typescript", - "--frontend", - "svelte", - "--backend", - "hono", - "--runtime", - "workers", - "--api", - "orpc", - "--database", - "sqlite", - "--orm", - "drizzle", - "--db-setup", - "d1", - "--auth", - "better-auth", - "--validation", - "valibot", - "--css-framework", - "tailwind", - "--ui-library", - "none", - "--web-deploy", - "cloudflare", - "--server-deploy", - "cloudflare", - "--payments", - "none", - "--email", - "none", - "--file-upload", - "none", - "--file-storage", - "none", - "--logging", - "none", - "--observability", - "none", - "--feature-flags", - "none", - "--analytics", - "none", - "--effect", - "none", - "--state-management", - "none", - "--forms", - "none", - "--testing", - "none", - "--ai", - "none", - "--realtime", - "none", - "--job-queue", - "none", - "--animation", - "none", - "--cms", - "none", - "--caching", - "none", - "--rate-limit", - "none", - "--i18n", - "none", - "--search", - "none", - "--vector-db", - "none", - "--addons", - "turborepo", - "--examples", - "none", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "typescript", - "frontend": [ - "svelte" - ], - "backend": "hono", - "runtime": "workers", - "api": "orpc", - "database": "sqlite", - "orm": "drizzle", - "dbSetup": "d1", - "auth": "better-auth", - "validation": "valibot", - "cssFramework": "tailwind", - "webDeploy": "cloudflare", - "serverDeploy": "cloudflare" - }, - "expectedAddons": [ - "turborepo" - ], - "strictMarkers": [ - { - "id": "frontend:svelte", - "deps": [ - "@sveltejs/kit" - ] - }, - { - "id": "backend:hono", - "deps": [ - "hono" - ] - }, - { - "id": "api:orpc", - "deps": [ - "@orpc/server" - ], - "source": [ - "@orpc/" - ] - }, - { - "id": "runtime:workers", - "deps": [ - "wrangler" - ] - }, - { - "id": "orm:drizzle", - "deps": [ - "drizzle-orm" - ], - "source": [ - "drizzle-orm" - ] - }, - { - "id": "db:d1", - "textAny": [ - "d1_databases", - "D1Database", - "d1-http" - ] - }, - { - "id": "auth:better-auth", - "deps": [ - "better-auth" - ], - "source": [ - "better-auth" - ] - }, - { - "id": "validation:valibot", - "deps": [ - "valibot" - ] - }, - { - "id": "css:tailwind", - "deps": [ - "tailwindcss" - ] - }, - { - "id": "adapter:cloudflare", - "deps": [ - "@sveltejs/adapter-cloudflare" - ] - }, - { - "id": "forbidden:trpc", - "forbiddenDeps": [ - "@trpc/server", - "@trpc/client" - ] - }, - { - "id": "forbidden:next", - "forbiddenDeps": [ - "next" - ] - }, - { - "id": "forbidden:adapter-node", - "forbiddenDeps": [ - "@sveltejs/adapter-node" - ] - } - ], - "validationProfile": { - "packageManager": "bun" - } - }, - { - "id": "dotnet-blazor-cqrs", - "introducedAt": "2026-08-21", - "title": ".NET Blazor app with Dapper, Duende IdentityServer, and HotChocolate GraphQL", - "lane": "core", - "difficulty": 2, - "family": "dotnet", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a .NET project using ASP.NET Blazor (not Minimal API or MVC).", - "Use Dapper for data access (not EF Core).", - "Use Duende IdentityServer for auth (not ASP.NET Identity).", - "Use HotChocolate for a GraphQL API (not Minimal API or gRPC).", - "Use PostgreSQL.", - "Use NUnit with Moq and Testcontainers for .NET (not xUnit).", - "Use Quartz.NET for background jobs (not Hangfire).", - "Use SignalR for realtime.", - "Use OpenTelemetry, NLog, and health checks for observability (not Serilog).", - "Use FluentValidation, Redis caching, and Docker deploy output." - ], - "naturalPrompt": "Build a .NET starter for an internal operations console. It needs a C# web UI, lightweight data access, a dedicated identity server, a GraphQL API, Postgres, background scheduling, realtime updates, validation, caching, observability, and container output. Choose the right .NET libraries rather than the framework defaults.", - "rightLibraryNotes": [ - "Blazor is required for the web framework.", - "Dapper is required for data access (not EF Core).", - "Duende IdentityServer is required for auth (not ASP.NET Identity).", - "HotChocolate GraphQL is required (not Minimal API), with NUnit and Quartz.NET." - ], - "canonicalFlags": [ - "--ecosystem", - "dotnet", - "--database", - "postgres", - "--dotnet-web-framework", - "aspnet-blazor", - "--dotnet-orm", - "dapper", - "--dotnet-auth", - "duende-identityserver", - "--dotnet-api", - "graphql-hotchocolate", - "--dotnet-testing", - "nunit", - "moq", - "testcontainers-dotnet", - "--dotnet-job-queue", - "quartz-net", - "--dotnet-realtime", - "signalr", - "--dotnet-observability", - "opentelemetry-dotnet", - "nlog", - "health-checks", - "--dotnet-validation", - "fluentvalidation", - "--dotnet-caching", - "redis", - "--dotnet-deploy", - "docker", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "claude-md", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "dotnet", - "database": "postgres", - "dotnetWebFramework": "aspnet-blazor", - "dotnetOrm": "dapper", - "dotnetAuth": "duende-identityserver", - "dotnetApi": "graphql-hotchocolate", - "dotnetTesting": [ - "nunit", - "moq", - "testcontainers-dotnet" - ], - "dotnetJobQueue": "quartz-net", - "dotnetRealtime": "signalr", - "dotnetObservability": [ - "opentelemetry-dotnet", - "nlog", - "health-checks" - ], - "dotnetValidation": "fluentvalidation", - "dotnetCaching": "redis", - "dotnetDeploy": "docker" - }, - "strictMarkers": [ - { - "id": "dotnet:blazor", - "text": [ - "RazorComponents" - ] - }, - { - "id": "orm:dapper", - "text": [ - "Dapper" - ] - }, - { - "id": "auth:duende", - "text": [ - "Duende" - ] - }, - { - "id": "api:hotchocolate", - "text": [ - "HotChocolate" - ] - }, - { - "id": "testing:nunit", - "text": [ - "NUnit" - ] - }, - { - "id": "testing:moq", - "text": [ - "Moq" - ] - }, - { - "id": "testing:testcontainers", - "text": [ - "Testcontainers" - ] - }, - { - "id": "jobs:quartz", - "text": [ - "Quartz" - ] - }, - { - "id": "realtime:signalr", - "text": [ - "SignalR" - ] - }, - { - "id": "validation:fluentvalidation", - "text": [ - "FluentValidation" - ] - }, - { - "id": "observability:nlog", - "text": [ - "NLog" - ] - }, - { - "id": "observability:opentelemetry", - "textAny": [ - "OpenTelemetry" - ] - }, - { - "id": "observability:health-checks", - "textAny": [ - "AddHealthChecks", - "MapHealthChecks", - "HealthChecks" - ] - }, - { - "id": "db:postgres", - "textAny": [ - "Npgsql" - ] - }, - { - "id": "caching:redis", - "textAny": [ - "StackExchange.Redis", - "AddStackExchangeRedisCache", - "Redis" - ] - }, - { - "id": "deploy:docker", - "files": [ - "Dockerfile" - ] - }, - { - "id": "forbidden:hangfire", - "forbiddenText": [ - "Hangfire" - ] - }, - { - "id": "forbidden:serilog", - "forbiddenText": [ - "Serilog" - ] - }, - { - "id": "forbidden:ef-core", - "forbiddenText": [ - "Microsoft.EntityFrameworkCore" - ] - }, - { - "id": "forbidden:aspnet-identity", - "forbiddenText": [ - "Microsoft.AspNetCore.Identity" - ] - }, - { - "id": "forbidden:grpc", - "forbiddenText": [ - "Grpc.AspNetCore", - "Grpc.Net.Client" - ] - }, - { - "id": "forbidden:xunit", - "forbiddenText": [ - "xunit" - ] - }, - { - "id": "forbidden:mvc", - "forbiddenText": [ - "AddControllersWithViews", - "MapControllerRoute", - "AddMvc(" - ] - }, - { - "id": "forbidden:minimal-api-host", - "forbiddenText": [ - "CreateSlimBuilder" - ] - } - ], - "validationProfile": { - "native": [ - "dotnet" - ] - } - }, - { - "id": "multi-ts-go-grpc", - "introducedAt": "2026-08-21", - "title": "Multi-ecosystem app: Nuxt (Vue) frontend with a Go Chi + gRPC backend", - "lane": "core", - "difficulty": 2, - "family": "multi-ecosystem", - "supportedByBetterFullstack": true, - "requirements": [ - "Create one multi-ecosystem project graph for a live auction dashboard: a Vue web frontend over a Go bid-processing backend.", - "Use a Nuxt (Vue) TypeScript frontend with Tailwind.", - "Use a Go backend with the Chi router (not Gin/Echo/Fiber).", - "Use sqlc for data access (not GORM or Ent).", - "Use gRPC-Go for typed service contracts.", - "Use goth for auth, Centrifuge for realtime bid updates, Watermill for messaging, Ristretto for caching, koanf for config, and zerolog for logging.", - "Use OpenTelemetry and Testify + GoMock.", - "Use PostgreSQL as the shared database." - ], - "naturalPrompt": "Build a multi-ecosystem starter for a live auction dashboard: a Vue/Nuxt web frontend and a Go backend. The Go side needs a lightweight router, type-safe SQL, typed gRPC contracts, social auth, scalable realtime, a messaging abstraction, an in-process cache, config management, structured logging, tracing, and test doubles. Use the project graph instead of one ecosystem.", - "rightLibraryNotes": [ - "The frontend must be TypeScript Nuxt (Vue).", - "The Go backend must use Chi, sqlc, and gRPC-Go.", - "Centrifuge and Watermill are required for realtime and messaging.", - "Ristretto, koanf, and zerolog are required (not Redis, Viper, zap)." - ], - "canonicalFlags": [ - "--part", - "frontend:typescript:nuxt", - "--part", - "frontend.css:typescript:tailwind", - "--part", - "backend:go:chi", - "--part", - "backend.orm:go:sqlc", - "--part", - "backend.api:go:grpc-go", - "--part", - "backend.auth:go:goth", - "--part", - "backend.logging:go:zerolog", - "--part", - "backend.realtime:go:centrifuge", - "--part", - "backend.jobQueue:go:watermill", - "--part", - "backend.caching:go:ristretto", - "--part", - "backend.config:go:koanf", - "--part", - "backend.observability:go:opentelemetry", - "--part", - "backend.testing:go:testify", - "--part", - "backend.testing:go:gomock", - "--part", - "database:universal:postgres", - "--addons", - "turborepo", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedParts": [ - "frontend:typescript:nuxt", - "frontend.css:typescript:tailwind", - "backend:go:chi", - "backend.orm:go:sqlc", - "backend.api:go:grpc-go", - "backend.auth:go:goth", - "backend.logging:go:zerolog", - "backend.realtime:go:centrifuge", - "backend.jobQueue:go:watermill", - "backend.caching:go:ristretto", - "backend.config:go:koanf", - "backend.observability:go:opentelemetry", - "backend.testing:go:testify", - "backend.testing:go:gomock", - "database:universal:postgres" - ], - "expectedAddons": [ - "turborepo" - ], - "strictMarkers": [ - { - "id": "frontend:nuxt", - "deps": [ - "nuxt" - ] - }, - { - "id": "frontend:tailwind", - "deps": [ - "tailwindcss" - ] - }, - { - "id": "backend:chi", - "text": [ - "github.com/go-chi/chi" - ] - }, - { - "id": "orm:sqlc", - "files": [ - "sqlc.*" - ] - }, - { - "id": "db:postgres", - "textAny": [ - "lib/pq", - "jackc/pgx", - "postgres" - ] - }, - { - "id": "api:grpc-go", - "text": [ - "google.golang.org/grpc" - ] - }, - { - "id": "auth:goth", - "text": [ - "github.com/markbates/goth" - ] - }, - { - "id": "realtime:centrifuge", - "text": [ - "github.com/centrifugal/centrifuge" - ] - }, - { - "id": "queue:watermill", - "text": [ - "github.com/ThreeDotsLabs/watermill" - ] - }, - { - "id": "caching:ristretto", - "text": [ - "github.com/dgraph-io/ristretto" - ] - }, - { - "id": "config:koanf", - "text": [ - "github.com/knadh/koanf" - ] - }, - { - "id": "logging:zerolog", - "text": [ - "github.com/rs/zerolog" - ] - }, - { - "id": "observability:opentelemetry", - "text": [ - "go.opentelemetry.io/otel" - ] - }, - { - "id": "testing:testify+gomock", - "text": [ - "github.com/stretchr/testify", - "go.uber.org/mock" - ] - }, - { - "id": "forbidden:gin", - "forbiddenText": [ - "github.com/gin-gonic/gin" - ] - }, - { - "id": "forbidden:echo", - "forbiddenText": [ - "github.com/labstack/echo" - ] - }, - { - "id": "forbidden:fiber", - "forbiddenText": [ - "github.com/gofiber/fiber" - ] - }, - { - "id": "forbidden:gorm", - "forbiddenText": [ - "gorm.io/gorm" - ] - }, - { - "id": "forbidden:ent", - "forbiddenText": [ - "entgo.io/ent" - ] - }, - { - "id": "forbidden:viper", - "forbiddenText": [ - "github.com/spf13/viper" - ] - }, - { - "id": "forbidden:redis", - "forbiddenText": [ - "github.com/redis/go-redis", - "github.com/go-redis/redis" - ] - }, - { - "id": "forbidden:zap", - "forbiddenText": [ - "go.uber.org/zap" - ] - } - ], - "validationProfile": { - "packageManager": "bun", - "native": [ - "go" - ] - } - }, - { - "id": "java-spring-jooq-keycloak", - "introducedAt": "2026-08-21", - "title": "Java Spring Boot API with jOOQ, Keycloak, GraphQL, and property/architecture tests", - "lane": "core", - "difficulty": 1, - "family": "java", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a Java project using Spring Boot with the Maven build tool.", - "Use jOOQ for data access (NOT Spring Data JPA).", - "Use Keycloak as the identity provider. Do not hand-roll authentication and do not use Spring Security as the identity provider (no in-app user store, form login, or `UserDetailsService`); wiring Keycloak through Spring Security's OAuth2 resource server is the expected integration.", - "Use Spring for GraphQL for the API and Logback for logging.", - "Use PostgreSQL.", - "Include MapStruct, Resilience4j, Spring for Kafka, Spring Batch, Micrometer Prometheus, Caffeine, springdoc-openapi, OpenTelemetry, Spring Validation, and Spring Actuator.", - "Include JUnit 5, Mockito, Testcontainers, AssertJ, REST Assured, WireMock, Awaitility, ArchUnit, and jqwik for testing." - ], - "naturalPrompt": "Build a Java Spring Boot starter for an event-driven service. It needs Postgres data access with a type-safe SQL layer, a dedicated identity server for auth, a GraphQL API, fault tolerance, event streaming, batch jobs, metrics, mapping, API docs, and tracing, plus a serious test stack with mocks, containers, HTTP stubs, architecture rules, and property-based tests. Choose the right Java libraries rather than the Spring defaults.", - "rightLibraryNotes": [ - "jOOQ is required for data access; Spring Data JPA is not used.", - "Keycloak is required as the identity provider; an in-app Spring Security user store or hand-rolled auth is a failure, while Spring Security's OAuth2 resource server is the expected way to validate Keycloak tokens.", - "Spring for GraphQL is required for the API.", - "ArchUnit and jqwik are required (architecture + property-based testing)." - ], - "canonicalFlags": [ - "--ecosystem", - "java", - "--database", - "postgres", - "--java-web-framework", - "spring-boot", - "--java-build-tool", - "maven", - "--java-orm", - "jooq", - "--java-auth", - "keycloak", - "--java-api", - "spring-graphql", - "--java-logging", - "logback", - "--java-libraries", - "mapstruct", - "resilience4j", - "spring-kafka", - "spring-batch", - "micrometer-prometheus", - "caffeine", - "springdoc-openapi", - "opentelemetry-java", - "spring-validation", - "spring-actuator", - "--java-testing-libraries", - "junit5", - "mockito", - "testcontainers", - "assertj", - "rest-assured", - "wiremock", - "awaitility", - "archunit", - "jqwik", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "claude-md", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "java", - "database": "postgres", - "javaWebFramework": "spring-boot", - "javaBuildTool": "maven", - "javaOrm": "jooq", - "javaAuth": "keycloak", - "javaApi": "spring-graphql", - "javaLogging": "logback", - "javaLibraries": [ - "mapstruct", - "resilience4j", - "spring-kafka", - "spring-batch", - "micrometer-prometheus", - "caffeine", - "springdoc-openapi", - "opentelemetry-java", - "spring-validation", - "spring-actuator" - ], - "javaTestingLibraries": [ - "junit5", - "mockito", - "testcontainers", - "assertj", - "rest-assured", - "wiremock", - "awaitility", - "archunit", - "jqwik" - ] - }, - "strictMarkers": [ - { - "id": "backend:spring-boot", - "text": [ - "spring-boot-starter-parent" - ] - }, - { - "id": "build:maven", - "files": [ - "pom.xml" - ] - }, - { - "id": "orm:jooq", - "text": [ - "jooq" - ] - }, - { - "id": "auth:keycloak", - "textAny": [ - "keycloak", - "Keycloak", - "KEYCLOAK" - ] - }, - { - "id": "auth:oauth2-resource-server", - "textAny": [ - "oauth2-resource-server", - "oauth2ResourceServer", - "issuer-uri", - "issuerUri" - ] - }, - { - "id": "api:spring-graphql", - "text": [ - "spring-boot-starter-graphql" - ] - }, - { - "id": "logging:logback", - "textAny": [ - "logback" - ] - }, - { - "id": "lib:mapstruct", - "text": [ - "mapstruct" - ] - }, - { - "id": "lib:resilience4j", - "text": [ - "resilience4j" - ] - }, - { - "id": "lib:spring-kafka", - "text": [ - "spring-kafka" - ] - }, - { - "id": "lib:spring-batch", - "text": [ - "spring-boot-starter-batch" - ] - }, - { - "id": "lib:micrometer-prometheus", - "text": [ - "micrometer-registry-prometheus" - ] - }, - { - "id": "lib:caffeine", - "textAny": [ - "caffeine" - ] - }, - { - "id": "lib:springdoc-openapi", - "textAny": [ - "springdoc" - ] - }, - { - "id": "lib:opentelemetry", - "textAny": [ - "opentelemetry" - ] - }, - { - "id": "lib:spring-validation", - "textAny": [ - "spring-boot-starter-validation" - ] - }, - { - "id": "lib:spring-actuator", - "textAny": [ - "spring-boot-starter-actuator" - ] - }, - { - "id": "testing:junit5", - "textAny": [ - "junit-jupiter", - "spring-boot-starter-test" - ] - }, - { - "id": "testing:mockito", - "textAny": [ - "mockito" - ] - }, - { - "id": "testing:assertj", - "textAny": [ - "assertj" - ] - }, - { - "id": "testing:rest-assured", - "textAny": [ - "rest-assured", - "restassured" - ] - }, - { - "id": "testing:wiremock", - "textAny": [ - "wiremock", - "WireMock" - ] - }, - { - "id": "testing:awaitility", - "textAny": [ - "awaitility" - ] - }, - { - "id": "testing:archunit", - "text": [ - "archunit" - ] - }, - { - "id": "testing:jqwik", - "text": [ - "jqwik" - ] - }, - { - "id": "testing:testcontainers", - "text": [ - "testcontainers" - ] - }, - { - "id": "forbidden:jpa", - "forbiddenText": [ - "spring-boot-starter-data-jpa" - ] - }, - { - "id": "forbidden:in-app-auth", - "forbiddenText": [ - "UserDetailsService", - "formLogin", - "InMemoryUserDetailsManager" - ] - } - ], - "validationProfile": { - "native": [ - "java" - ] - } - }, - { - "id": "elixir-broadway-absinthe", - "introducedAt": "2026-08-21", - "title": "Elixir Phoenix LiveView app with Absinthe, Broadway, Oban, and Nx", - "lane": "core", - "difficulty": 2, - "family": "elixir", - "supportedByBetterFullstack": true, - "requirements": [ - "Create an Elixir project using Phoenix LiveView (not plain Phoenix).", - "Use Ecto SQL with PostgreSQL and Ecto changesets for validation.", - "Use Guardian for auth. Do not generate or hand-write a phx.gen.auth user store (no bcrypt password hashing, no UserSessionController) and do not use Ueberauth.", - "Use Absinthe for a GraphQL API.", - "Include Broadway and Nx as libraries.", - "Use Phoenix Presence for realtime, Oban for jobs, Finch as the HTTP client, Jason for JSON, Swoosh for email, Nebulex for caching, and PromEx for observability.", - "Use Wallaby for testing, Dialyxir for code quality (not Credo), and Fly for deploy output." - ], - "naturalPrompt": "Build an Elixir Phoenix starter for a realtime data-ingestion app. It needs server-rendered live views, Postgres via Ecto, a dedicated JWT auth library, a GraphQL API, data pipelines, numerical/ML support, presence tracking, durable background jobs, a pooled HTTP client, caching, Prometheus metrics, browser-based tests, static analysis, and a deploy target. Pick the right BEAM libraries rather than the framework defaults.", - "rightLibraryNotes": [ - "Phoenix LiveView is required (not plain Phoenix).", - "Guardian is required for auth; Absinthe is required for the GraphQL API.", - "Broadway and Oban are required for pipelines and jobs.", - "Presence, Finch, Nebulex, PromEx, Wallaby, and Dialyxir are the required choices." - ], - "canonicalFlags": [ - "--ecosystem", - "elixir", - "--database", - "postgres", - "--elixir-web-framework", - "phoenix-live-view", - "--elixir-orm", - "ecto-sql", - "--elixir-auth", - "guardian", - "--elixir-api", - "absinthe", - "--elixir-libraries", - "broadway", - "nx", - "--elixir-realtime", - "presence", - "--elixir-jobs", - "oban", - "--elixir-validation", - "ecto-changesets", - "--elixir-http", - "finch", - "--elixir-json", - "jason", - "--elixir-email", - "swoosh", - "--elixir-caching", - "nebulex", - "--elixir-observability", - "prom_ex", - "--elixir-testing", - "wallaby", - "--elixir-quality", - "dialyxir", - "--elixir-deploy", - "fly", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "claude-md", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "elixir", - "database": "postgres", - "elixirWebFramework": "phoenix-live-view", - "elixirOrm": "ecto-sql", - "elixirAuth": "guardian", - "elixirApi": "absinthe", - "elixirLibraries": [ - "broadway", - "nx" - ], - "elixirRealtime": "presence", - "elixirJobs": "oban", - "elixirValidation": "ecto-changesets", - "elixirHttp": "finch", - "elixirJson": "jason", - "elixirEmail": "swoosh", - "elixirCaching": "nebulex", - "elixirObservability": "prom_ex", - "elixirTesting": "wallaby", - "elixirQuality": "dialyxir", - "elixirDeploy": "fly" - }, - "strictMarkers": [ - { - "id": "web:phoenix-live-view", - "text": [ - "phoenix_live_view" - ] - }, - { - "id": "orm:ecto-sql", - "text": [ - "ecto_sql" - ] - }, - { - "id": "auth:guardian", - "text": [ - ":guardian" - ] - }, - { - "id": "api:absinthe", - "text": [ - "absinthe" - ] - }, - { - "id": "lib:broadway", - "text": [ - "broadway" - ] - }, - { - "id": "lib:nx", - "text": [ - "{:nx," - ] - }, - { - "id": "jobs:oban", - "text": [ - ":oban" - ] - }, - { - "id": "http:finch", - "text": [ - ":finch" - ] - }, - { - "id": "caching:nebulex", - "text": [ - "nebulex" - ] - }, - { - "id": "observability:prom_ex", - "text": [ - "prom_ex" - ] - }, - { - "id": "testing:wallaby", - "text": [ - "wallaby" - ] - }, - { - "id": "quality:dialyxir", - "text": [ - "dialyxir" - ] - }, - { - "id": "realtime:presence", - "textAny": [ - "Presence" - ] - }, - { - "id": "validation:ecto-changesets", - "textAny": [ - "changeset" - ] - }, - { - "id": "json:jason", - "textAny": [ - ":jason", - "Jason" - ] - }, - { - "id": "email:swoosh", - "textAny": [ - ":swoosh", - "Swoosh" - ] - }, - { - "id": "deploy:fly", - "files": [ - "fly.toml" - ] - }, - { - "id": "forbidden:credo", - "forbiddenText": [ - ":credo" - ] - }, - { - "id": "forbidden:ueberauth", - "forbiddenText": [ - "ueberauth" - ] - }, - { - "id": "forbidden:phx-gen-auth", - "forbiddenText": [ - ":bcrypt_elixir", - "UserSessionController", - "phx.gen.auth" - ] - } - ], - "validationProfile": { - "native": [ - "elixir" - ] - } - }, - { - "id": "react-native-expo", - "introducedAt": "2026-08-21", - "title": "React Native Expo habit tracker with Expo Router, Uniwind, MMKV, and Maestro + RNTL", - "lane": "core", - "difficulty": 2, - "family": "react-native", - "supportedByBetterFullstack": true, - "requirements": [ - "Create a React Native (Expo) habit-tracker app: a habit list, a detail screen, and a settings screen, all working offline.", - "Use Expo Router for navigation.", - "Use Uniwind for Tailwind-style styling. NativeWind is the familiar answer and the wrong one here, do not add it.", - "Use MMKV for on-device storage; all habit data lives on the device.", - "Use Maestro plus React Native Testing Library for testing.", - "Use Expo Notifications for habit reminders, Expo Updates for OTA, and Expo Linking for deep linking into a habit's detail screen.", - "This is a mobile-only project: no backend, database, sync service, or auth." - ], - "naturalPrompt": "Build a React Native habit tracker on Expo that works fully offline: a habit list, detail and settings screens, local reminders, deep links into a habit, and over-the-air updates. It needs file-based navigation, Tailwind-style styling, fast on-device key-value storage, and both end-to-end and unit testing. It is a standalone mobile app with no server, database, or accounts.", - "rightLibraryNotes": [ - "Expo Router is required for navigation.", - "Uniwind is the required styling approach (native-uniwind frontend); NativeWind is a failure.", - "MMKV is required for storage; Maestro + RNTL for testing.", - "Expo Notifications / Updates / Linking are the required push / OTA / deep-linking choices." - ], - "canonicalFlags": [ - "--ecosystem", - "react-native", - "--frontend", - "native-uniwind", - "--auth", - "none", - "--mobile-navigation", - "expo-router", - "--mobile-ui", - "uniwind", - "--mobile-storage", - "mmkv", - "--mobile-testing", - "maestro-react-native-testing-library", - "--mobile-push", - "expo-notifications", - "--mobile-ota", - "expo-updates", - "--mobile-deep-linking", - "expo-linking", - "--ai-docs", - "claude-md", - "--package-manager", - "bun", - "--no-install", - "--no-git", - "--disable-analytics" - ], - "expectedConfig": { - "ecosystem": "react-native", - "frontend": [ - "native-uniwind" - ], - "mobileNavigation": "expo-router", - "mobileUI": "uniwind", - "mobileStorage": "mmkv", - "mobileTesting": "maestro-react-native-testing-library", - "mobilePush": "expo-notifications", - "mobileOTA": "expo-updates", - "mobileDeepLinking": "expo-linking" - }, - "strictMarkers": [ - { - "id": "nav:expo-router", - "deps": [ - "expo-router" - ] - }, - { - "id": "styling:uniwind", - "deps": [ - "uniwind" - ] - }, - { - "id": "storage:mmkv", - "deps": [ - "react-native-mmkv" - ] - }, - { - "id": "push:expo-notifications", - "deps": [ - "expo-notifications" - ] - }, - { - "id": "ota:expo-updates", - "deps": [ - "expo-updates" - ] - }, - { - "id": "deep-linking:expo-linking", - "deps": [ - "expo-linking" - ] - }, - { - "id": "testing:rntl", - "deps": [ - "@testing-library/react-native" - ] - }, - { - "id": "testing:maestro", - "files": [ - ".maestro/*.y*ml" - ] - }, - { - "id": "forbidden:nativewind", - "forbiddenDeps": [ - "nativewind" - ] - }, - { - "id": "forbidden:backend", - "forbiddenDeps": [ - "express", - "hono", - "fastify", - "elysia", - "@nestjs/core", - "convex" - ] - }, - { - "id": "forbidden:database", - "forbiddenDeps": [ - "drizzle-orm", - "@prisma/client", - "@supabase/supabase-js", - "firebase", - "@react-native-firebase/app" - ] - }, - { - "id": "forbidden:sync", - "forbiddenDeps": [ - "@powersync/react-native", - "@instantdb/react-native", - "replicache", - "@rocicorp/zero", - "@nozbe/watermelondb" - ] - }, - { - "id": "forbidden:auth", - "forbiddenDeps": [ - "better-auth", - "@clerk/clerk-expo", - "expo-auth-session", - "@react-native-google-signin/google-signin" - ] - } - ], - "validationProfile": { - "packageManager": "bun" - } - }, - { - "id": "frontier-polyglot-proto", - "introducedAt": "2026-08-21", - "title": "Frontier: polyglot monorepo, shared protobuf across a Rust gRPC service, a Go gateway, and a TS client", - "lane": "core", - "difficulty": 3, - "family": "multi-ecosystem", - "supportedByBetterFullstack": false, - "paths": [ - "prompt" - ], - "requirements": [ - "Create one monorepo with a single shared Protocol Buffers (proto3) service contract.", - "Implement the core service in Rust using Tonic for gRPC.", - "Implement an edge gateway in Go that speaks gRPC to the Rust service and exposes HTTP/JSON.", - "Implement a TypeScript web client generated from the same proto contract.", - "Wire codegen so all three consume the one .proto definition; provide build scripts per package." - ], - "naturalPrompt": "Build a polyglot monorepo around a single service contract: a Rust gRPC core service, a Go gateway that bridges gRPC to HTTP/JSON, and a TypeScript client, all generated from one shared Protocol Buffers definition. Set up the codegen and per-package builds so the three stay in sync.", - "rightLibraryNotes": [ - "A single shared proto3 contract must drive all three languages.", - "Rust uses Tonic for the gRPC service; Go uses grpc-go for the gateway.", - "The TypeScript client must be generated from the same proto." - ], - "canonicalFlags": [], - "strictMarkers": [ - { - "id": "proto:proto3", - "text": [ - "proto3" - ] - }, - { - "id": "proto:contract-file", - "files": [ - "*.proto" - ] - }, - { - "id": "rust:tonic", - "text": [ - "tonic" - ] - }, - { - "id": "go:grpc", - "text": [ - "google.golang.org/grpc" - ] - }, - { - "id": "ts:protobuf", - "text": [ - "protobuf" - ] - }, - { - "id": "codegen:from-proto", - "textAny": [ - "buf.gen", - "protoc", - "tonic_build", - "protoc-gen" - ] - }, - { - "id": "rust:proto-generated", - "textAny": [ - "prost", - "tonic_build", - "include_proto" - ] - }, - { - "id": "go:proto-generated", - "textAny": [ - "google.golang.org/protobuf", - "protoc-gen-go", - ".pb.go" - ] - }, - { - "id": "ts:proto-generated", - "textAny": [ - "ts-proto", - "@bufbuild/protobuf", - "@connectrpc/connect", - "google-protobuf", - "protobufjs", - "_pb.ts", - "_pb.js" - ] - }, - { - "id": "gateway:http-json", - "textAny": [ - "net/http", - "grpc-gateway", - "gin-gonic", - "go-chi" - ] - } - ], - "prerequisiteCommands": [ - { - "command": [ - "buf", - "generate" - ], - "whenConfigFound": [ - "buf.gen.yaml", - "buf.gen.yml", - "buf.gen.json" - ] - } - ], - "validationProfile": { - "packageManager": "bun", - "native": [ - "cargo", - "go" - ] - } - }, - { - "id": "frontier-effect-eventsourcing", - "introducedAt": "2026-08-21", - "title": "Frontier: TypeScript Effect service with event-sourcing/CQRS and tRPC-over-WebSocket subscriptions", - "lane": "core", - "difficulty": 3, - "family": "typescript", - "supportedByBetterFullstack": false, - "paths": [ - "prompt" - ], - "requirements": [ - "Create a TypeScript backend for a bank-ledger service built on the Effect ecosystem (effect runtime, services, layers).", - "Implement event-sourcing with CQRS: an append-only event store, write-side command handlers (open account, deposit, withdraw with overdraft rejection), and read-side balance projections.", - "Projections must be rebuildable by replaying the event store from zero, and applying an event twice must not corrupt a projection.", - "Expose the API via tRPC, including a subscription over WebSockets that streams balance updates from the read model.", - "Include an outbox pattern for reliable event publication.", - "Provide build and type-check scripts." - ], - "naturalPrompt": "Build a TypeScript bank-ledger backend on the Effect ecosystem that uses event sourcing with CQRS, an append-only event store, command handlers for open/deposit/withdraw with overdraft rejection on the write side, replayable idempotent balance projections on the read side, and an outbox for reliable publishing. Expose it through tRPC, including a WebSocket subscription that streams balance updates.", - "rightLibraryNotes": [ - "The service layer must be built on Effect.", - "Use event-sourcing + CQRS (event store, projections, outbox), not plain CRUD.", - "Projections must be replayable and idempotent.", - "Expose tRPC with a WebSocket subscription for the read model." - ], - "canonicalFlags": [], - "strictMarkers": [ - { - "id": "runtime:effect", - "deps": [ - "effect" - ] - }, - { - "id": "api:trpc", - "deps": [ - "@trpc/server" - ] - }, - { - "id": "ws:subscription", - "text": [ - "subscription" - ] - }, - { - "id": "pattern:event-sourcing", - "text": [ - "projection" - ] - }, - { - "id": "store:event-store", - "textAny": [ - "eventStore", - "EventStore", - "event_store" - ] - }, - { - "id": "store:append-only", - "textAny": [ - "append" - ] - }, - { - "id": "cqrs:commands", - "textAny": [ - "command", - "Command" - ] - }, - { - "id": "command:open-account", - "textAny": [ - "openAccount", - "OpenAccount", - "open_account" - ] - }, - { - "id": "command:deposit", - "textAny": [ - "deposit", - "Deposit" - ] - }, - { - "id": "command:withdraw", - "textAny": [ - "withdraw", - "Withdraw" - ] - }, - { - "id": "ledger:overdraft", - "textAny": [ - "overdraft", - "Overdraft", - "insufficient", - "Insufficient" - ] - }, - { - "id": "projection:replay", - "textAny": [ - "replay", - "Replay", - "rebuild", - "Rebuild" - ] - }, - { - "id": "projection:idempotent", - "textAny": [ - "idempot", - "Idempot" - ] - }, - { - "id": "pattern:outbox", - "textAny": [ - "outbox", - "Outbox" - ] - } - ], - "validationProfile": { - "packageManager": "bun" - } - } - ], - "aggregates": { - "bySpecCell": [ - { - "key": "java-spring-jooq-keycloak|gemini-3.7-flash|low|prompt", - "specId": "java-spring-jooq-keycloak", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 1, - "qualityScoredRuns": 1, - "qualityPassRate": 100, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 100, - "specScore": 100, - "avgDurationMs": 207073, - "medianDurationMs": 207073, - "p95DurationMs": 207073, - "avgLines": 1132, - "failureTags": {}, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "dotnet-blazor-cqrs|gemini-3.7-flash|low|prompt", - "specId": "dotnet-blazor-cqrs", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 1, - "qualityScoredRuns": 1, - "qualityPassRate": 100, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 100, - "specScore": 100, - "avgDurationMs": 291181, - "medianDurationMs": 291181, - "p95DurationMs": 291181, - "avgLines": 1916, - "failureTags": {}, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "elixir-broadway-absinthe|gemini-3.7-flash|low|prompt", - "specId": "elixir-broadway-absinthe", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 1, - "qualityScoredRuns": 1, - "qualityPassRate": 100, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 100, - "specScore": 100, - "avgDurationMs": 558890, - "medianDurationMs": 558890, - "p95DurationMs": 558890, - "avgLines": 3806, - "failureTags": { - "test-failed": 1 - }, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "go-realtime-api|gemini-3.7-flash|low|prompt", - "specId": "go-realtime-api", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 90, - "specScore": 90, - "avgDurationMs": 136043, - "medianDurationMs": 136043, - "p95DurationMs": 136043, - "avgLines": 9261, - "failureTags": { - "format-failed": 1, - "test-failed": 1 - }, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "frontier-polyglot-proto|gemini-3.7-flash|low|prompt", - "specId": "frontier-polyglot-proto", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 87, - "specScore": 87, - "avgDurationMs": 233198, - "medianDurationMs": 233198, - "p95DurationMs": 233198, - "avgLines": 2023, - "failureTags": { - "format-failed": 1, - "lint-failed": 1, - "test-failed": 1 - }, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "multi-dotnet-ops|gemini-3.7-flash|low|prompt", - "specId": "multi-dotnet-ops", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 94, - "commandDisciplinePercent": 100, - "index": 84, - "specScore": 84, - "avgDurationMs": 222737, - "medianDurationMs": 222737, - "p95DurationMs": 222737, - "avgLines": 2162, - "failureTags": { - "format-failed": 1, - "lint-failed": 1, - "stack-mismatch": 1, - "test-failed": 1 - }, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "frontier-effect-eventsourcing|gemini-3.7-flash|low|prompt", - "specId": "frontier-effect-eventsourcing", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 80, - "specScore": 80, - "avgDurationMs": 140036, - "medianDurationMs": 140036, - "p95DurationMs": 140036, - "avgLines": 991, - "failureTags": { - "format-failed": 1, - "lint-failed": 1, - "test-failed": 1 - }, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "rust-leptos-axum|gemini-3.7-flash|low|prompt", - "specId": "rust-leptos-axum", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 1, - "passRate": 100, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 21, - "high": 100 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 100, - "passAnySpecs": 1, - "passAllSpecs": 1, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 80, - "specScore": 80, - "avgDurationMs": 452864, - "medianDurationMs": 452864, - "p95DurationMs": 452864, - "avgLines": 1328, - "failureTags": { - "format-failed": 1, - "lint-failed": 1, - "test-failed": 1 - }, - "outcomeCounts": { - "success": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "python-ingestion-api|gemini-3.7-flash|low|prompt", - "specId": "python-ingestion-api", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 0, - "passRate": 0, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 0, - "high": 79 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 0, - "passAnySpecs": 0, - "passAllSpecs": 0, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 20, - "specScore": 20, - "avgDurationMs": 76121, - "medianDurationMs": 76121, - "p95DurationMs": 76121, - "avgLines": 1008, - "failureTags": { - "build-failed": 1, - "format-failed": 1, - "install-failed": 1, - "lint-failed": 1, - "test-failed": 1, - "typecheck-failed": 1, - "validation-failed": 1 - }, - "outcomeCounts": { - "model-failure": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "react-native-expo|gemini-3.7-flash|low|prompt", - "specId": "react-native-expo", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 0, - "passRate": 0, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 0, - "high": 79 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 0, - "passAnySpecs": 0, - "passAllSpecs": 0, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 20, - "specScore": 20, - "avgDurationMs": 258778, - "medianDurationMs": 258778, - "p95DurationMs": 258778, - "avgLines": 1500, - "failureTags": { - "build-failed": 1, - "format-failed": 1, - "lint-failed": 1, - "test-failed": 1, - "typecheck-failed": 1, - "validation-failed": 1 - }, - "outcomeCounts": { - "model-failure": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "ts-svelte-edge-orpc|gemini-3.7-flash|low|prompt", - "specId": "ts-svelte-edge-orpc", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 0, - "passRate": 0, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 0, - "high": 79 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 0, - "passAnySpecs": 0, - "passAllSpecs": 0, - "stackPercent": 100, - "commandDisciplinePercent": 100, - "index": 20, - "specScore": 20, - "avgDurationMs": 526396, - "medianDurationMs": 526396, - "p95DurationMs": 526396, - "avgLines": 220342, - "failureTags": { - "build-failed": 1, - "format-failed": 1, - "lint-failed": 1, - "test-failed": 1, - "validation-failed": 1 - }, - "outcomeCounts": { - "model-failure": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "multi-ts-go-grpc|gemini-3.7-flash|low|prompt", - "specId": "multi-ts-go-grpc", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 0, - "passRate": 0, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 0, - "high": 79 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 0, - "passAnySpecs": 0, - "passAllSpecs": 0, - "stackPercent": 91, - "commandDisciplinePercent": 100, - "index": 18, - "specScore": 18, - "avgDurationMs": 233437, - "medianDurationMs": 233437, - "p95DurationMs": 233437, - "avgLines": 9804, - "failureTags": { - "build-failed": 1, - "format-failed": 1, - "install-failed": 1, - "lint-failed": 1, - "stack-mismatch": 1, - "test-failed": 1, - "validation-failed": 1 - }, - "outcomeCounts": { - "model-failure": 1 - }, - "publicationEligibility": "ranked" - }, - { - "key": "ai-search-workbench|gemini-3.7-flash|low|prompt", - "specId": "ai-search-workbench", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 1, - "scoredRuns": 1, - "inconclusiveCount": 0, - "passCount": 0, - "passRate": 0, - "qualityPassCount": 0, - "qualityScoredRuns": 1, - "qualityPassRate": 0, - "passCi95": { - "low": 0, - "high": 79 - }, - "ciReportable": false, - "specCount": 1, - "macroPassRate": 0, - "passAnySpecs": 0, - "passAllSpecs": 0, - "stackPercent": 85, - "commandDisciplinePercent": 100, - "index": 17, - "specScore": 17, - "avgDurationMs": 90681, - "medianDurationMs": 90681, - "p95DurationMs": 90681, - "avgLines": 1796, - "failureTags": { - "build-failed": 1, - "format-failed": 1, - "install-failed": 1, - "lint-failed": 1, - "stack-mismatch": 1, - "test-failed": 1, - "typecheck-failed": 1, - "validation-failed": 1 - }, - "outcomeCounts": { - "model-failure": 1 - }, - "publicationEligibility": "ranked" - } - ], - "leaderboard": [ - { - "key": "gemini-3.7-flash|low|prompt", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "runs": 13, - "scoredRuns": 13, - "inconclusiveCount": 0, - "passCount": 8, - "passRate": 62, - "qualityPassCount": 3, - "qualityScoredRuns": 13, - "qualityPassRate": 23, - "passCi95": { - "low": 36, - "high": 82 - }, - "ciReportable": true, - "specCount": 13, - "macroPassRate": 62, - "passAnySpecs": 8, - "passAllSpecs": 8, - "stackPercent": 98, - "commandDisciplinePercent": 100, - "index": 63, - "specScore": 63, - "avgDurationMs": 263649, - "medianDurationMs": 233198, - "p95DurationMs": 558890, - "avgLines": 19775, - "failureTags": { - "test-failed": 11, - "build-failed": 5, - "format-failed": 10, - "install-failed": 3, - "lint-failed": 9, - "typecheck-failed": 3, - "validation-failed": 5, - "stack-mismatch": 3 - }, - "outcomeCounts": { - "success": 8, - "model-failure": 5 - }, - "publicationEligibility": "ranked" - } - ] - }, - "results": [ - { - "id": "elixir-broadway-absinthe-gemini-3.7-flash-low-prompt-r01", - "specId": "elixir-broadway-absinthe", - "specTitle": "Elixir Phoenix LiveView app with Absinthe, Broadway, Oban, and Nx", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/elixir-broadway-absinthe-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-elixir-broadway-absinthe-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/elixir-broadway-absinthe-gemini-3.7-flash-low-prompt-r01/sb21-elixir-broadway-absinthe-prompt-low", - "codeMetrics": { - "files": 67, - "lines": 3806, - "bytes": 138537 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 558890, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "mix deps.get", - "exitCode": 0, - "timedOut": false, - "durationMs": 12728, - "stdoutTail": "a 8592), pack-reused 0 (from 0) \nResolving Hex dependencies...\nResolution completed in 0.426s\nUnchanged:\n absinthe 1.11.0\n absinthe_plug 1.5.10\n bandit 1.12.5\n broadway 1.3.0\n castore 1.0.21\n cc_precompiler 0.1.11\n certifi 2.17.0\n complex 0.7.0\n db_connection 2.10.2\n decimal 3.1.1\n dialyxir 1.4.7\n dns_cluster 0.2.0\n ecto 3.14.2\n ecto_sql 3.14.0\n elixir_make 0.10.0\n erlex 0.2.9\n esbuild 0.10.0\n expo 1.1.1\n file_system 1.1.1\n finch 0.23.0\n fine 0.1.6\n gen_stage 1.3.2\n gettext 1.0.2\n guardian 2.5.0\n h2 0.12.0\n hackney 4.7.4\n hpax 1.0.4\n httpoison 3.0.0\n idna 7.1.0\n jason 1.4.5\n jose 1.11.12\n lazy_html 0.1.12\n mime 2.0.7\n mimerl 1.5.0\n mint 1.9.3\n nebulex 2.6.6\n nimble_options 1.1.1\n nimble_parsec 1.4.2\n nimble_pool 1.1.0\n nx 0.13.1\n oban 2.24.0\n octo_fetch 0.5.0\n parse_trans 3.4.2\n peep 4.4.0\n phoenix 1.8.13\n phoenix_ecto 4.7.0\n phoenix_html 4.3.0\n phoenix_live_dashboard 0.8.7\n phoenix_live_reload 1.7.0\n phoenix_live_view 1.2.11\n phoenix_pubsub 2.3.0\n phoenix_template 1.0.4\n plug 1.20.3\n plug_crypto 2.2.0\n postgrex 0.22.4\n prom_ex 1.12.0\n quic 1.8.1\n req 0.7.4\n ssl_verify_fun 1.1.7\n swoosh 1.28.0\n tailwind 0.5.1\n telemetry 1.4.2\n telemetry_metrics 1.2.0\n telemetry_metrics_prometheus_core 1.2.1\n telemetry_poller 1.3.0\n tesla 1.21.2\n thousand_island 1.5.0\n wallaby 0.31.0\n web_driver_client 0.3.0\n websock 0.5.3\n websock_adapter 0.6.0\n webtransport 0.4.5\n* Getting phoenix (Hex package)\n* Getting phoenix_ecto (Hex package)\n* Getting ecto_sql (Hex package)\n* Getting postgrex (Hex package)\n* Getting phoenix_html (Hex package)\n* Getting phoenix_live_reload (Hex package)\n* Getting phoenix_live_view (Hex package)\n* Getting lazy_html (Hex package)\n* Getting phoenix_live_dashboard (Hex package)\n* Getting esbuild (Hex package)\n* Getting tailwind (Hex package)\n* Getting swoosh (Hex package)\n* Getting req (Hex package)\n* Getting telemetry_metrics (Hex package)\n* Getting telemetry_poller (Hex package)\n* Getting gettext (Hex package)\n* Getting jason (Hex package)\n* Getting dns_cluster (Hex package)\n* Getting bandit (Hex package)\n* Getting guardian (Hex package)\n* Getting absinthe (Hex package)\n* Getting absinthe_plug (Hex package)\n* Getting broadway (Hex package)\n* Getting nx (Hex package)\n* Getting oban (Hex package)\n* Getting finch (Hex package)\n* Getting nebulex (Hex package)\n* Getting prom_ex (Hex package)\n* Getting wallaby (Hex package)\n* Getting dialyxir (Hex package)\n* Getting erlex (Hex package)\n* Getting httpoison (Hex package)\n* Getting web_driver_client (Hex package)\n* Getting hackney (Hex package)\n* Getting tesla (Hex package)\n* Getting mime (Hex package)\n* Getting certifi (Hex package)\n* Getting h2 (Hex package)\n* Getting idna (Hex package)\n* Getting mimerl (Hex package)\n* Getting parse_trans (Hex package)\n* Getting ssl_verify_fun (Hex package)\n* Getting webtransport (Hex package)\n* Getting quic (Hex package)\n* Getting octo_fetch (Hex package)\n* Getting peep (Hex package)\n* Getting telemetry (Hex package)\n* Getting telemetry_metrics_prometheus_core (Hex package)\n* Getting nimble_options (Hex package)\n* Getting castore (Hex package)\n* Getting mint (Hex package)\n* Getting nimble_pool (Hex package)\n* Getting hpax (Hex package)\n* Getting complex (Hex package)\n* Getting gen_stage (Hex package)\n* Getting plug (Hex package)\n* Getting plug_crypto (Hex package)\n* Getting nimble_parsec (Hex package)\n* Getting jose (Hex package)\n* Getting thousand_island (Hex package)\n* Getting websock (Hex package)\n* Getting expo (Hex package)\n* Getting cc_precompiler (Hex package)\n* Getting elixir_make (Hex package)\n* Getting fine (Hex package)\n* Getting phoenix_template (Hex package)\n* Getting file_system (Hex package)\n* Getting db_connection (Hex package)\n* Getting decimal (Hex package)\n* Getting ecto (Hex package)\n* Getting phoenix_pubsub (Hex package)\n* Getting websock_adapter (Hex package)\nYou have added/upgraded packages you could sponsor, run `mix hex.sponsor` to learn more\n", - "stderrTail": "", - "startedAtMs": 1787851787121, - "lastActivityAtMs": 1787851799737 - }, - "build": { - "command": "mix compile", - "exitCode": 0, - "timedOut": false, - "durationMs": 91442, - "stdoutTail": " | \t ^\n\nsrc/jwa/jose_jwa.erl:262:7: Warning: 'catch ...' is deprecated; please use 'try ... catch ... end' instead.\nCompile directive 'nowarn_deprecated_catch' can be used to suppress\nwarnings in selected modules.\n% 262| \tcase catch ?MAYBE_START_JOSE(ets:lookup_element(?TAB, xchacha20_poly1305_module, 2)) of\n% | \t ^\n\nCompiling 8 files (.ex)\nGenerated jose app\n==> ssl_verify_fun\nCompiling 7 files (.erl)\nGenerated ssl_verify_fun app\n==> complex\nCompiling 2 files (.ex)\nGenerated complex app\n==> erlex\nCompiling 1 file (.xrl)\nCompiling 1 file (.yrl)\nCompiling 2 files (.erl)\nCompiling 2 files (.ex)\nGenerated erlex app\n==> castore\nCompiling 1 file (.ex)\nGenerated castore app\n==> octo_fetch\nCompiling 3 files (.ex)\nGenerated octo_fetch app\n==> mint\nCompiling 1 file (.erl)\nCompiling 19 files (.ex)\nGenerated mint app\n==> decimal\nCompiling 4 files (.ex)\nGenerated decimal app\n==> jason\nCompiling 10 files (.ex)\nGenerated jason app\n==> esbuild\nCompiling 4 files (.ex)\nGenerated esbuild app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling idna\n==> nimble_parsec\nCompiling 4 files (.ex)\nGenerated nimble_parsec app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling telemetry\n==> telemetry_metrics\nCompiling 7 files (.ex)\nGenerated telemetry_metrics app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling telemetry_poller\n==> absinthe\nCompiling 1 file (.yrl)\nCompiling 1 file (.erl)\nCompiling 264 files (.ex)\nGenerated absinthe app\n==> broadway\nCompiling 22 files (.ex)\nGenerated broadway app\n==> nx\nCompiling 43 files (.ex)\nGenerated nx app\n==> nebulex\nCompiling 45 files (.ex)\nGenerated nebulex app\n==> telemetry_metrics_prometheus_core\nCompiling 9 files (.ex)\nGenerated telemetry_metrics_prometheus_core app\n==> thousand_island\nCompiling 18 files (.ex)\nGenerated thousand_island app\n==> db_connection\nCompiling 18 files (.ex)\nGenerated db_connection app\n==> ecto\nCompiling 56 files (.ex)\nGenerated ecto app\n==> phoenix_html\nCompiling 6 files (.ex)\nGenerated phoenix_html app\n==> phoenix_template\nCompiling 4 files (.ex)\nGenerated phoenix_template app\n==> expo\nCompiling 2 files (.erl)\nCompiling 22 files (.ex)\nGenerated expo app\n==> gettext\nCompiling 18 files (.ex)\nGenerated gettext app\n==> phoenix_pubsub\nCompiling 12 files (.ex)\nGenerated phoenix_pubsub app\n==> dns_cluster\nCompiling 1 file (.ex)\nGenerated dns_cluster app\n==> dialyxir\nCompiling 67 files (.ex)\nGenerated dialyxir app\n==> plug\nCompiling 1 file (.erl)\nCompiling 42 files (.ex)\nGenerated plug app\n==> guardian\nCompiling 25 files (.ex)\nGenerated guardian app\n==> absinthe_plug\nCompiling 18 files (.ex)\nGenerated absinthe_plug app\n==> postgrex\nCompiling 70 files (.ex)\nGenerated postgrex app\n==> phoenix_ecto\nCompiling 7 files (.ex)\nGenerated phoenix_ecto app\n==> ecto_sql\nCompiling 25 files (.ex)\nGenerated ecto_sql app\n==> oban\nCompiling 69 files (.ex)\nGenerated oban app\n==> peep\nCompiling 18 files (.ex)\nGenerated peep app\n==> nimble_pool\nCompiling 2 files (.ex)\nGenerated nimble_pool app\n==> finch\nCompiling 23 files (.ex)\nGenerated finch app\n==> req\nCompiling 24 files (.ex)\nGenerated req app\n==> tailwind\nCompiling 3 files (.ex)\nGenerated tailwind app\n==> websock\nCompiling 1 file (.ex)\nGenerated websock app\n==> bandit\nCompiling 54 files (.ex)\nGenerated bandit app\n==> swoosh\nCompiling 62 files (.ex)\nGenerated swoosh app\n==> websock_adapter\nCompiling 4 files (.ex)\nGenerated websock_adapter app\n==> phoenix\nCompiling 74 files (.ex)\nGenerated phoenix app\n==> phoenix_live_reload\nCompiling 5 files (.ex)\nGenerated phoenix_live_reload app\n==> phoenix_live_view\nCompiling 55 files (.ex)\nGenerated phoenix_live_view app\n==> phoenix_live_dashboard\nCompiling 36 files (.ex)\nGenerated phoenix_live_dashboard app\n==> prom_ex\nCompiling 40 files (.ex)\nGenerated prom_ex app\n==> sb21_elixir_broadway_absinthe_prompt_low\nCompiling 31 files (.ex)\nGenerated sb21_elixir_broadway_absinthe_prompt_low app\n", - "stderrTail": "ndant:\n\n defp ensure_empty_msgstr!(%Expo.Message.Plural{} = message)\n\n it has type:\n\n dynamic(%Expo.Message.Plural{})\n\n previous clauses have already matched on the following types:\n\n %Expo.Message.Singular{}\n %Expo.Message.Plural{}\n\n │\n 391 │ defp ensure_empty_msgstr!(%Message.Plural{} = message) do\n │ ~\n │\n └─ lib/gettext/extractor.ex:391:8: Gettext.Extractor.ensure_empty_msgstr!/1\n\n warning: Kernel.ParallelCompiler.async/1 is deprecated. Use `pmap/2` instead\n │\n 426 │ Kernel.ParallelCompiler.async(fn ->\n │ ~\n │\n └─ lib/gettext/compiler.ex:426:39: Gettext.Compiler.compile_po_files/3\n\n warning: comparison between distinct types found:\n\n net_state.started != :no\n\n given types:\n\n dynamic(not :no) != :no\n\n where \"net_state\" was given the types:\n\n # type: dynamic()\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(not false and not nil)\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(%{..., started: term()})\n # from: lib/dns_cluster.ex:210:25\n net_state.started == :no\n\n # type: dynamic(%{..., started: not :no})\n # from: lib/dns_cluster.ex:220:25\n net_state.started == :no\n\n While Elixir can compare across all types, you are comparing across types which are always disjoint, and the result is either always true or always false\n\n type warning found at:\n │\n 221 │ (!release? and net_state.started != :no and net_state[:name_domain] != :longnames) ->\n │ ~\n │\n └─ lib/dns_cluster.ex:221:44: DNSCluster.warn_on_invalid_dist/0\n\n warning: unused require Logger\n │\n 123 │ require Logger\n │ ~\n │\n └─ lib/absinthe/plug.ex:123:3\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\n warning: Plug.Cowboy.child_spec/1 is undefined (module Plug.Cowboy is not available or is yet to be defined)\n │\n 556 │ Plug.Cowboy.child_spec(\n │ ~\n │\n └─ lib/prom_ex.ex:556:19: PromEx.metrics_server_child_spec/4\n\n", - "startedAtMs": 1787851799850, - "lastActivityAtMs": 1787851891188 - }, - "format": { - "command": "mix format --check-formatted", - "exitCode": 0, - "timedOut": false, - "durationMs": 688, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787851891292, - "lastActivityAtMs": 1787851891292 - }, - "test": { - "command": "mix test", - "exitCode": 1, - "timedOut": false, - "durationMs": 112139, - "stdoutTail": "ling 22 files (.ex)\nGenerated expo app\n==> gettext\nCompiling 18 files (.ex)\nGenerated gettext app\n==> phoenix_pubsub\nCompiling 12 files (.ex)\nGenerated phoenix_pubsub app\n==> dns_cluster\nCompiling 1 file (.ex)\nGenerated dns_cluster app\n==> dialyxir\nCompiling 67 files (.ex)\nGenerated dialyxir app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling certifi\n==> plug\nCompiling 1 file (.erl)\nCompiling 42 files (.ex)\nGenerated plug app\n==> guardian\nCompiling 25 files (.ex)\nGenerated guardian app\n==> absinthe_plug\nCompiling 18 files (.ex)\nGenerated absinthe_plug app\n==> postgrex\nCompiling 70 files (.ex)\nGenerated postgrex app\n==> phoenix_ecto\nCompiling 7 files (.ex)\nGenerated phoenix_ecto app\n==> ecto_sql\nCompiling 25 files (.ex)\nGenerated ecto_sql app\n==> oban\nCompiling 69 files (.ex)\nGenerated oban app\n==> peep\nCompiling 18 files (.ex)\nGenerated peep app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling parse_trans\n==> nimble_pool\nCompiling 2 files (.ex)\nGenerated nimble_pool app\n==> finch\nCompiling 23 files (.ex)\nGenerated finch app\n==> req\nCompiling 24 files (.ex)\nGenerated req app\n==> cc_precompiler\nCompiling 3 files (.ex)\nGenerated cc_precompiler app\n==> lazy_html\nCompiling 3 files (.ex)\nGenerated lazy_html app\n==> tailwind\nCompiling 3 files (.ex)\nGenerated tailwind app\n==> websock\nCompiling 1 file (.ex)\nGenerated websock app\n==> bandit\nCompiling 54 files (.ex)\nGenerated bandit app\n==> websock_adapter\nCompiling 4 files (.ex)\nGenerated websock_adapter app\n==> phoenix\nCompiling 74 files (.ex)\nGenerated phoenix app\n==> phoenix_live_view\nCompiling 55 files (.ex)\nGenerated phoenix_live_view app\n==> phoenix_live_dashboard\nCompiling 36 files (.ex)\nGenerated phoenix_live_dashboard app\n==> prom_ex\nCompiling 40 files (.ex)\nGenerated prom_ex app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling quic\n===> Analyzing applications...\n===> Compiling webtransport\n===> Analyzing applications...\n===> Compiling hackney\n==> swoosh\nCompiling 62 files (.ex)\nGenerated swoosh app\n==> httpoison\nCompiling 3 files (.ex)\nGenerated httpoison app\n==> tesla\nCompiling 57 files (.ex)\nGenerated tesla app\n==> web_driver_client\nCompiling 97 files (.ex)\nGenerated web_driver_client app\n==> wallaby\nCompiling 29 files (.ex)\nGenerated wallaby app\n==> sb21_elixir_broadway_absinthe_prompt_low\nCompiling 33 files (.ex)\nGenerated sb21_elixir_broadway_absinthe_prompt_low app\n\n20:33:24.084 [error] Postgrex.Protocol (#PID<0.12714.0> (\"db_conn_1\")) failed to connect: ** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n\n20:33:24.087 [error] :gen_statem #PID<0.12714.0> terminating\n** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n (db_connection 2.10.2) lib/db_connection/connection.ex:121: DBConnection.Connection.handle_event/4\n (stdlib 8.0.3) gen_statem.erl:3743: :gen_statem.loop_state_callback/11\n (stdlib 8.0.3) proc_lib.erl:333: :proc_lib.init_p_do_apply/3\nProcess Label: \"db_conn_1\"\nQueue: [internal: {:connect, :init}]\nPostponed: []\nState: Postgrex.Protocol\nCallback mode: :handle_event_function, state_enter: false\n\n20:33:24.091 [error] Postgrex.Protocol (#PID<0.12720.0> (\"db_conn_1\")) failed to connect: ** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n\n20:33:24.091 [error] :gen_statem #PID<0.12720.0> terminating\n** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n (db_connection 2.10.2) lib/db_connection/connection.ex:121: DBConnection.Connection.handle_event/4\n (stdlib 8.0.3) gen_statem.erl:3743: :gen_statem.loop_state_callback/11\n (stdlib 8.0.3) proc_lib.erl:333: :proc_lib.init_p_do_apply/3\nProcess Label: \"db_conn_1\"\nQueue: [internal: {:connect, :init}]\nPostponed: []\nState: Postgrex.Protocol\nCallback mode: :handle_event_function, state_enter: false\n", - "stderrTail": " │\n └─ lib/gettext/extractor.ex:391:8: Gettext.Extractor.ensure_empty_msgstr!/1\n\n warning: Kernel.ParallelCompiler.async/1 is deprecated. Use `pmap/2` instead\n │\n 426 │ Kernel.ParallelCompiler.async(fn ->\n │ ~\n │\n └─ lib/gettext/compiler.ex:426:39: Gettext.Compiler.compile_po_files/3\n\n warning: comparison between distinct types found:\n\n net_state.started != :no\n\n given types:\n\n dynamic(not :no) != :no\n\n where \"net_state\" was given the types:\n\n # type: dynamic()\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(not false and not nil)\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(%{..., started: term()})\n # from: lib/dns_cluster.ex:210:25\n net_state.started == :no\n\n # type: dynamic(%{..., started: not :no})\n # from: lib/dns_cluster.ex:220:25\n net_state.started == :no\n\n While Elixir can compare across all types, you are comparing across types which are always disjoint, and the result is either always true or always false\n\n type warning found at:\n │\n 221 │ (!release? and net_state.started != :no and net_state[:name_domain] != :longnames) ->\n │ ~\n │\n └─ lib/dns_cluster.ex:221:44: DNSCluster.warn_on_invalid_dist/0\n\n warning: unused require Logger\n │\n 123 │ require Logger\n │ ~\n │\n └─ lib/absinthe/plug.ex:123:3\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\n warning: Plug.Cowboy.child_spec/1 is undefined (module Plug.Cowboy is not available or is yet to be defined)\n │\n 556 │ Plug.Cowboy.child_spec(\n │ ~\n │\n └─ lib/prom_ex.ex:556:19: PromEx.metrics_server_child_spec/4\n\n warning: this clause of defp test_paths/1 is never used (or it will always fail/warn when invoked)\n │\n 92 │ defp test_paths(driver) when driver in @drivers, do: [\"integration_test/#{driver}\"]\n │ ~\n │\n └─ mix.exs:92:8: Wallaby.Mixfile.test_paths/1\n\n** (Mix) The database for Sb21ElixirBroadwayAbsinthePromptLow.Repo couldn't be created: killed\n", - "startedAtMs": 1787851891980, - "lastActivityAtMs": 1787852004093 - } - }, - "install": { - "command": "mix deps.get", - "exitCode": 0, - "timedOut": false, - "durationMs": 12728, - "stdoutTail": "a 8592), pack-reused 0 (from 0) \nResolving Hex dependencies...\nResolution completed in 0.426s\nUnchanged:\n absinthe 1.11.0\n absinthe_plug 1.5.10\n bandit 1.12.5\n broadway 1.3.0\n castore 1.0.21\n cc_precompiler 0.1.11\n certifi 2.17.0\n complex 0.7.0\n db_connection 2.10.2\n decimal 3.1.1\n dialyxir 1.4.7\n dns_cluster 0.2.0\n ecto 3.14.2\n ecto_sql 3.14.0\n elixir_make 0.10.0\n erlex 0.2.9\n esbuild 0.10.0\n expo 1.1.1\n file_system 1.1.1\n finch 0.23.0\n fine 0.1.6\n gen_stage 1.3.2\n gettext 1.0.2\n guardian 2.5.0\n h2 0.12.0\n hackney 4.7.4\n hpax 1.0.4\n httpoison 3.0.0\n idna 7.1.0\n jason 1.4.5\n jose 1.11.12\n lazy_html 0.1.12\n mime 2.0.7\n mimerl 1.5.0\n mint 1.9.3\n nebulex 2.6.6\n nimble_options 1.1.1\n nimble_parsec 1.4.2\n nimble_pool 1.1.0\n nx 0.13.1\n oban 2.24.0\n octo_fetch 0.5.0\n parse_trans 3.4.2\n peep 4.4.0\n phoenix 1.8.13\n phoenix_ecto 4.7.0\n phoenix_html 4.3.0\n phoenix_live_dashboard 0.8.7\n phoenix_live_reload 1.7.0\n phoenix_live_view 1.2.11\n phoenix_pubsub 2.3.0\n phoenix_template 1.0.4\n plug 1.20.3\n plug_crypto 2.2.0\n postgrex 0.22.4\n prom_ex 1.12.0\n quic 1.8.1\n req 0.7.4\n ssl_verify_fun 1.1.7\n swoosh 1.28.0\n tailwind 0.5.1\n telemetry 1.4.2\n telemetry_metrics 1.2.0\n telemetry_metrics_prometheus_core 1.2.1\n telemetry_poller 1.3.0\n tesla 1.21.2\n thousand_island 1.5.0\n wallaby 0.31.0\n web_driver_client 0.3.0\n websock 0.5.3\n websock_adapter 0.6.0\n webtransport 0.4.5\n* Getting phoenix (Hex package)\n* Getting phoenix_ecto (Hex package)\n* Getting ecto_sql (Hex package)\n* Getting postgrex (Hex package)\n* Getting phoenix_html (Hex package)\n* Getting phoenix_live_reload (Hex package)\n* Getting phoenix_live_view (Hex package)\n* Getting lazy_html (Hex package)\n* Getting phoenix_live_dashboard (Hex package)\n* Getting esbuild (Hex package)\n* Getting tailwind (Hex package)\n* Getting swoosh (Hex package)\n* Getting req (Hex package)\n* Getting telemetry_metrics (Hex package)\n* Getting telemetry_poller (Hex package)\n* Getting gettext (Hex package)\n* Getting jason (Hex package)\n* Getting dns_cluster (Hex package)\n* Getting bandit (Hex package)\n* Getting guardian (Hex package)\n* Getting absinthe (Hex package)\n* Getting absinthe_plug (Hex package)\n* Getting broadway (Hex package)\n* Getting nx (Hex package)\n* Getting oban (Hex package)\n* Getting finch (Hex package)\n* Getting nebulex (Hex package)\n* Getting prom_ex (Hex package)\n* Getting wallaby (Hex package)\n* Getting dialyxir (Hex package)\n* Getting erlex (Hex package)\n* Getting httpoison (Hex package)\n* Getting web_driver_client (Hex package)\n* Getting hackney (Hex package)\n* Getting tesla (Hex package)\n* Getting mime (Hex package)\n* Getting certifi (Hex package)\n* Getting h2 (Hex package)\n* Getting idna (Hex package)\n* Getting mimerl (Hex package)\n* Getting parse_trans (Hex package)\n* Getting ssl_verify_fun (Hex package)\n* Getting webtransport (Hex package)\n* Getting quic (Hex package)\n* Getting octo_fetch (Hex package)\n* Getting peep (Hex package)\n* Getting telemetry (Hex package)\n* Getting telemetry_metrics_prometheus_core (Hex package)\n* Getting nimble_options (Hex package)\n* Getting castore (Hex package)\n* Getting mint (Hex package)\n* Getting nimble_pool (Hex package)\n* Getting hpax (Hex package)\n* Getting complex (Hex package)\n* Getting gen_stage (Hex package)\n* Getting plug (Hex package)\n* Getting plug_crypto (Hex package)\n* Getting nimble_parsec (Hex package)\n* Getting jose (Hex package)\n* Getting thousand_island (Hex package)\n* Getting websock (Hex package)\n* Getting expo (Hex package)\n* Getting cc_precompiler (Hex package)\n* Getting elixir_make (Hex package)\n* Getting fine (Hex package)\n* Getting phoenix_template (Hex package)\n* Getting file_system (Hex package)\n* Getting db_connection (Hex package)\n* Getting decimal (Hex package)\n* Getting ecto (Hex package)\n* Getting phoenix_pubsub (Hex package)\n* Getting websock_adapter (Hex package)\nYou have added/upgraded packages you could sponsor, run `mix hex.sponsor` to learn more\n", - "stderrTail": "", - "startedAtMs": 1787851787121, - "lastActivityAtMs": 1787851799737 - }, - "build": { - "command": "mix compile", - "exitCode": 0, - "timedOut": false, - "durationMs": 91442, - "stdoutTail": " | \t ^\n\nsrc/jwa/jose_jwa.erl:262:7: Warning: 'catch ...' is deprecated; please use 'try ... catch ... end' instead.\nCompile directive 'nowarn_deprecated_catch' can be used to suppress\nwarnings in selected modules.\n% 262| \tcase catch ?MAYBE_START_JOSE(ets:lookup_element(?TAB, xchacha20_poly1305_module, 2)) of\n% | \t ^\n\nCompiling 8 files (.ex)\nGenerated jose app\n==> ssl_verify_fun\nCompiling 7 files (.erl)\nGenerated ssl_verify_fun app\n==> complex\nCompiling 2 files (.ex)\nGenerated complex app\n==> erlex\nCompiling 1 file (.xrl)\nCompiling 1 file (.yrl)\nCompiling 2 files (.erl)\nCompiling 2 files (.ex)\nGenerated erlex app\n==> castore\nCompiling 1 file (.ex)\nGenerated castore app\n==> octo_fetch\nCompiling 3 files (.ex)\nGenerated octo_fetch app\n==> mint\nCompiling 1 file (.erl)\nCompiling 19 files (.ex)\nGenerated mint app\n==> decimal\nCompiling 4 files (.ex)\nGenerated decimal app\n==> jason\nCompiling 10 files (.ex)\nGenerated jason app\n==> esbuild\nCompiling 4 files (.ex)\nGenerated esbuild app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling idna\n==> nimble_parsec\nCompiling 4 files (.ex)\nGenerated nimble_parsec app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling telemetry\n==> telemetry_metrics\nCompiling 7 files (.ex)\nGenerated telemetry_metrics app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling telemetry_poller\n==> absinthe\nCompiling 1 file (.yrl)\nCompiling 1 file (.erl)\nCompiling 264 files (.ex)\nGenerated absinthe app\n==> broadway\nCompiling 22 files (.ex)\nGenerated broadway app\n==> nx\nCompiling 43 files (.ex)\nGenerated nx app\n==> nebulex\nCompiling 45 files (.ex)\nGenerated nebulex app\n==> telemetry_metrics_prometheus_core\nCompiling 9 files (.ex)\nGenerated telemetry_metrics_prometheus_core app\n==> thousand_island\nCompiling 18 files (.ex)\nGenerated thousand_island app\n==> db_connection\nCompiling 18 files (.ex)\nGenerated db_connection app\n==> ecto\nCompiling 56 files (.ex)\nGenerated ecto app\n==> phoenix_html\nCompiling 6 files (.ex)\nGenerated phoenix_html app\n==> phoenix_template\nCompiling 4 files (.ex)\nGenerated phoenix_template app\n==> expo\nCompiling 2 files (.erl)\nCompiling 22 files (.ex)\nGenerated expo app\n==> gettext\nCompiling 18 files (.ex)\nGenerated gettext app\n==> phoenix_pubsub\nCompiling 12 files (.ex)\nGenerated phoenix_pubsub app\n==> dns_cluster\nCompiling 1 file (.ex)\nGenerated dns_cluster app\n==> dialyxir\nCompiling 67 files (.ex)\nGenerated dialyxir app\n==> plug\nCompiling 1 file (.erl)\nCompiling 42 files (.ex)\nGenerated plug app\n==> guardian\nCompiling 25 files (.ex)\nGenerated guardian app\n==> absinthe_plug\nCompiling 18 files (.ex)\nGenerated absinthe_plug app\n==> postgrex\nCompiling 70 files (.ex)\nGenerated postgrex app\n==> phoenix_ecto\nCompiling 7 files (.ex)\nGenerated phoenix_ecto app\n==> ecto_sql\nCompiling 25 files (.ex)\nGenerated ecto_sql app\n==> oban\nCompiling 69 files (.ex)\nGenerated oban app\n==> peep\nCompiling 18 files (.ex)\nGenerated peep app\n==> nimble_pool\nCompiling 2 files (.ex)\nGenerated nimble_pool app\n==> finch\nCompiling 23 files (.ex)\nGenerated finch app\n==> req\nCompiling 24 files (.ex)\nGenerated req app\n==> tailwind\nCompiling 3 files (.ex)\nGenerated tailwind app\n==> websock\nCompiling 1 file (.ex)\nGenerated websock app\n==> bandit\nCompiling 54 files (.ex)\nGenerated bandit app\n==> swoosh\nCompiling 62 files (.ex)\nGenerated swoosh app\n==> websock_adapter\nCompiling 4 files (.ex)\nGenerated websock_adapter app\n==> phoenix\nCompiling 74 files (.ex)\nGenerated phoenix app\n==> phoenix_live_reload\nCompiling 5 files (.ex)\nGenerated phoenix_live_reload app\n==> phoenix_live_view\nCompiling 55 files (.ex)\nGenerated phoenix_live_view app\n==> phoenix_live_dashboard\nCompiling 36 files (.ex)\nGenerated phoenix_live_dashboard app\n==> prom_ex\nCompiling 40 files (.ex)\nGenerated prom_ex app\n==> sb21_elixir_broadway_absinthe_prompt_low\nCompiling 31 files (.ex)\nGenerated sb21_elixir_broadway_absinthe_prompt_low app\n", - "stderrTail": "ndant:\n\n defp ensure_empty_msgstr!(%Expo.Message.Plural{} = message)\n\n it has type:\n\n dynamic(%Expo.Message.Plural{})\n\n previous clauses have already matched on the following types:\n\n %Expo.Message.Singular{}\n %Expo.Message.Plural{}\n\n │\n 391 │ defp ensure_empty_msgstr!(%Message.Plural{} = message) do\n │ ~\n │\n └─ lib/gettext/extractor.ex:391:8: Gettext.Extractor.ensure_empty_msgstr!/1\n\n warning: Kernel.ParallelCompiler.async/1 is deprecated. Use `pmap/2` instead\n │\n 426 │ Kernel.ParallelCompiler.async(fn ->\n │ ~\n │\n └─ lib/gettext/compiler.ex:426:39: Gettext.Compiler.compile_po_files/3\n\n warning: comparison between distinct types found:\n\n net_state.started != :no\n\n given types:\n\n dynamic(not :no) != :no\n\n where \"net_state\" was given the types:\n\n # type: dynamic()\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(not false and not nil)\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(%{..., started: term()})\n # from: lib/dns_cluster.ex:210:25\n net_state.started == :no\n\n # type: dynamic(%{..., started: not :no})\n # from: lib/dns_cluster.ex:220:25\n net_state.started == :no\n\n While Elixir can compare across all types, you are comparing across types which are always disjoint, and the result is either always true or always false\n\n type warning found at:\n │\n 221 │ (!release? and net_state.started != :no and net_state[:name_domain] != :longnames) ->\n │ ~\n │\n └─ lib/dns_cluster.ex:221:44: DNSCluster.warn_on_invalid_dist/0\n\n warning: unused require Logger\n │\n 123 │ require Logger\n │ ~\n │\n └─ lib/absinthe/plug.ex:123:3\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\n warning: Plug.Cowboy.child_spec/1 is undefined (module Plug.Cowboy is not available or is yet to be defined)\n │\n 556 │ Plug.Cowboy.child_spec(\n │ ~\n │\n └─ lib/prom_ex.ex:556:19: PromEx.metrics_server_child_spec/4\n\n", - "startedAtMs": 1787851799850, - "lastActivityAtMs": 1787851891188 - }, - "format": { - "command": "mix format --check-formatted", - "exitCode": 0, - "timedOut": false, - "durationMs": 688, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787851891292, - "lastActivityAtMs": 1787851891292 - }, - "test": { - "command": "mix test", - "exitCode": 1, - "timedOut": false, - "durationMs": 112139, - "stdoutTail": "ling 22 files (.ex)\nGenerated expo app\n==> gettext\nCompiling 18 files (.ex)\nGenerated gettext app\n==> phoenix_pubsub\nCompiling 12 files (.ex)\nGenerated phoenix_pubsub app\n==> dns_cluster\nCompiling 1 file (.ex)\nGenerated dns_cluster app\n==> dialyxir\nCompiling 67 files (.ex)\nGenerated dialyxir app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling certifi\n==> plug\nCompiling 1 file (.erl)\nCompiling 42 files (.ex)\nGenerated plug app\n==> guardian\nCompiling 25 files (.ex)\nGenerated guardian app\n==> absinthe_plug\nCompiling 18 files (.ex)\nGenerated absinthe_plug app\n==> postgrex\nCompiling 70 files (.ex)\nGenerated postgrex app\n==> phoenix_ecto\nCompiling 7 files (.ex)\nGenerated phoenix_ecto app\n==> ecto_sql\nCompiling 25 files (.ex)\nGenerated ecto_sql app\n==> oban\nCompiling 69 files (.ex)\nGenerated oban app\n==> peep\nCompiling 18 files (.ex)\nGenerated peep app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling parse_trans\n==> nimble_pool\nCompiling 2 files (.ex)\nGenerated nimble_pool app\n==> finch\nCompiling 23 files (.ex)\nGenerated finch app\n==> req\nCompiling 24 files (.ex)\nGenerated req app\n==> cc_precompiler\nCompiling 3 files (.ex)\nGenerated cc_precompiler app\n==> lazy_html\nCompiling 3 files (.ex)\nGenerated lazy_html app\n==> tailwind\nCompiling 3 files (.ex)\nGenerated tailwind app\n==> websock\nCompiling 1 file (.ex)\nGenerated websock app\n==> bandit\nCompiling 54 files (.ex)\nGenerated bandit app\n==> websock_adapter\nCompiling 4 files (.ex)\nGenerated websock_adapter app\n==> phoenix\nCompiling 74 files (.ex)\nGenerated phoenix app\n==> phoenix_live_view\nCompiling 55 files (.ex)\nGenerated phoenix_live_view app\n==> phoenix_live_dashboard\nCompiling 36 files (.ex)\nGenerated phoenix_live_dashboard app\n==> prom_ex\nCompiling 40 files (.ex)\nGenerated prom_ex app\n==> sb21_elixir_broadway_absinthe_prompt_low\n===> Analyzing applications...\n===> Compiling quic\n===> Analyzing applications...\n===> Compiling webtransport\n===> Analyzing applications...\n===> Compiling hackney\n==> swoosh\nCompiling 62 files (.ex)\nGenerated swoosh app\n==> httpoison\nCompiling 3 files (.ex)\nGenerated httpoison app\n==> tesla\nCompiling 57 files (.ex)\nGenerated tesla app\n==> web_driver_client\nCompiling 97 files (.ex)\nGenerated web_driver_client app\n==> wallaby\nCompiling 29 files (.ex)\nGenerated wallaby app\n==> sb21_elixir_broadway_absinthe_prompt_low\nCompiling 33 files (.ex)\nGenerated sb21_elixir_broadway_absinthe_prompt_low app\n\n20:33:24.084 [error] Postgrex.Protocol (#PID<0.12714.0> (\"db_conn_1\")) failed to connect: ** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n\n20:33:24.087 [error] :gen_statem #PID<0.12714.0> terminating\n** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n (db_connection 2.10.2) lib/db_connection/connection.ex:121: DBConnection.Connection.handle_event/4\n (stdlib 8.0.3) gen_statem.erl:3743: :gen_statem.loop_state_callback/11\n (stdlib 8.0.3) proc_lib.erl:333: :proc_lib.init_p_do_apply/3\nProcess Label: \"db_conn_1\"\nQueue: [internal: {:connect, :init}]\nPostponed: []\nState: Postgrex.Protocol\nCallback mode: :handle_event_function, state_enter: false\n\n20:33:24.091 [error] Postgrex.Protocol (#PID<0.12720.0> (\"db_conn_1\")) failed to connect: ** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n\n20:33:24.091 [error] :gen_statem #PID<0.12720.0> terminating\n** (DBConnection.ConnectionError) tcp connect (localhost:5432): connection refused - :econnrefused\n (db_connection 2.10.2) lib/db_connection/connection.ex:121: DBConnection.Connection.handle_event/4\n (stdlib 8.0.3) gen_statem.erl:3743: :gen_statem.loop_state_callback/11\n (stdlib 8.0.3) proc_lib.erl:333: :proc_lib.init_p_do_apply/3\nProcess Label: \"db_conn_1\"\nQueue: [internal: {:connect, :init}]\nPostponed: []\nState: Postgrex.Protocol\nCallback mode: :handle_event_function, state_enter: false\n", - "stderrTail": " │\n └─ lib/gettext/extractor.ex:391:8: Gettext.Extractor.ensure_empty_msgstr!/1\n\n warning: Kernel.ParallelCompiler.async/1 is deprecated. Use `pmap/2` instead\n │\n 426 │ Kernel.ParallelCompiler.async(fn ->\n │ ~\n │\n └─ lib/gettext/compiler.ex:426:39: Gettext.Compiler.compile_po_files/3\n\n warning: comparison between distinct types found:\n\n net_state.started != :no\n\n given types:\n\n dynamic(not :no) != :no\n\n where \"net_state\" was given the types:\n\n # type: dynamic()\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(not false and not nil)\n # from: lib/dns_cluster.ex:207:8\n net_state\n\n # type: dynamic(%{..., started: term()})\n # from: lib/dns_cluster.ex:210:25\n net_state.started == :no\n\n # type: dynamic(%{..., started: not :no})\n # from: lib/dns_cluster.ex:220:25\n net_state.started == :no\n\n While Elixir can compare across all types, you are comparing across types which are always disjoint, and the result is either always true or always false\n\n type warning found at:\n │\n 221 │ (!release? and net_state.started != :no and net_state[:name_domain] != :longnames) ->\n │ ~\n │\n └─ lib/dns_cluster.ex:221:44: DNSCluster.warn_on_invalid_dist/0\n\n warning: unused require Logger\n │\n 123 │ require Logger\n │ ~\n │\n └─ lib/absinthe/plug.ex:123:3\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\nwarning: \"xref: [exclude: ...]\" in your mix.exs file is deprecated, instead use: \"elixirc_options: [no_warn_undefined: ...]\"\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:243: Mix.Tasks.Compile.Elixir.xref_exclude_opts/2\n (mix 1.20.3) lib/mix/tasks/compile.elixir.ex:142: Mix.Tasks.Compile.Elixir.run/1\n (mix 1.20.3) lib/mix/task.ex:502: anonymous fn/3 in Mix.Task.run_task/5\n (mix 1.20.3) lib/mix/task.compiler.ex:299: Mix.Task.Compiler.run_compiler/2\n (mix 1.20.3) lib/mix/task.compiler.ex:287: Mix.Task.Compiler.run/4\n (mix 1.20.3) lib/mix/tasks/compile.all.ex:75: Mix.Tasks.Compile.All.do_run/2\n\n warning: Plug.Cowboy.child_spec/1 is undefined (module Plug.Cowboy is not available or is yet to be defined)\n │\n 556 │ Plug.Cowboy.child_spec(\n │ ~\n │\n └─ lib/prom_ex.ex:556:19: PromEx.metrics_server_child_spec/4\n\n warning: this clause of defp test_paths/1 is never used (or it will always fail/warn when invoked)\n │\n 92 │ defp test_paths(driver) when driver in @drivers, do: [\"integration_test/#{driver}\"]\n │ ~\n │\n └─ mix.exs:92:8: Wallaby.Mixfile.test_paths/1\n\n** (Mix) The database for Sb21ElixirBroadwayAbsinthePromptLow.Repo couldn't be created: killed\n", - "startedAtMs": 1787851891980, - "lastActivityAtMs": 1787852004093 - }, - "sourceHash": "5e5b90672ecce5cbd8f72d73cfcd907503be525a398a7c103b16a035ffe66032", - "cacheKey": "fc0bfe453386a646460bdfd61d833d15747fcb2aaf775edd72857de35efcd4e6", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 20, - "total": 20, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "test-failed" - ], - "outcome": "success" - }, - { - "id": "python-ingestion-api-gemini-3.7-flash-low-prompt-r01", - "specId": "python-ingestion-api", - "specTitle": "Python FastAPI ingestion API with AI, queues, realtime, and quality gates", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/python-ingestion-api-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-python-ingestion-api-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/python-ingestion-api-gemini-3.7-flash-low-prompt-r01/sb21-python-ingestion-api-prompt-low", - "codeMetrics": { - "files": 19, - "lines": 1008, - "bytes": 32464 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 76121, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "uv sync --all-extras", - "exitCode": 1, - "timedOut": false, - "durationMs": 2398, - "stdoutTail": "", - "stderrTail": "iB)\nDownloading asyncpg (3.4MiB)\nDownloading psycopg2-binary (4.1MiB)\nDownloading openai (1.6MiB)\n Downloaded openai\n × Failed to build `sb21-python-ingestion-api-prompt-low @\n │ file:///home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/python-ingestion-api-gemini-3.7-flash-low-prompt-r01/sb21-python-ingestion-api-prompt-low.validate-tmp`\n ├─▶ The build backend returned an error\n ╰─▶ Call to `hatchling.build.build_editable` failed (exit status: 1)\n\n [stderr]\n Traceback (most recent call last):\n File \"\", line 11, in \n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/build.py\",\n line 83, in build_editable\n return os.path.basename(next(builder.build(directory=wheel_directory,\n versions=[\"editable\"])))\n \n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/plugin/interface.py\",\n line 157, in build\n artifact = version_api[version](directory, **build_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 547, in build_editable\n return self.build_editable_detection(directory, **build_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 559, in build_editable_detection\n for included_file in self.recurse_selected_project_files():\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/plugin/interface.py\",\n line 182, in recurse_selected_project_files\n if self.config.only_include:\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/functools.py\", line 995, in __get__\n val = self.func(instance)\n ^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/config.py\",\n line 715, in only_include\n only_include = only_include_config.get(\"only-include\",\n self.default_only_include()) or self.packages\n \n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 293, in default_only_include\n return self.default_file_selection_options.only_include\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/functools.py\", line 995, in __get__\n val = self.func(instance)\n ^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 281, in default_file_selection_options\n raise ValueError(message)\n ValueError: Unable to determine which files to ship\n inside the wheel using the following heuristics:\n https://hatch.pypa.io/latest/plugins/builder/wheel/#default-file-selection\n\n The most likely cause of this is that there is no directory that matches\n the name of your project (sb21_python_ingestion_api_prompt_low).\n\n At least one file selection option must be defined\n in the `tool.hatch.build.targets.wheel` table, see:\n https://hatch.pypa.io/latest/config/build/\n\n As an example, if you intend to ship a directory named `foo` that\n resides within a `src` directory located at the root of your project,\n you can define the following:\n\n [tool.hatch.build.targets.wheel]\n packages = [\"src/foo\"]\n\n\nhint: Build failures usually indicate a problem with the package or the build environment\n", - "startedAtMs": 1787852004352, - "lastActivityAtMs": 1787852006743 - }, - "not-run:compile": { - "command": "compile not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:typecheck": { - "command": "typecheck not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "unvalidated:no-build-surface": { - "command": "no build or typecheck surface was discovered. A green install alone is not a pass", - "exitCode": 1, - "timedOut": false, - "status": "ran", - "durationMs": 0, - "stdoutTail": "", - "stderrTail": "no build or typecheck surface was discovered. A green install alone is not a pass" - } - }, - "install": { - "command": "uv sync --all-extras", - "exitCode": 1, - "timedOut": false, - "durationMs": 2398, - "stdoutTail": "", - "stderrTail": "iB)\nDownloading asyncpg (3.4MiB)\nDownloading psycopg2-binary (4.1MiB)\nDownloading openai (1.6MiB)\n Downloaded openai\n × Failed to build `sb21-python-ingestion-api-prompt-low @\n │ file:///home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/python-ingestion-api-gemini-3.7-flash-low-prompt-r01/sb21-python-ingestion-api-prompt-low.validate-tmp`\n ├─▶ The build backend returned an error\n ╰─▶ Call to `hatchling.build.build_editable` failed (exit status: 1)\n\n [stderr]\n Traceback (most recent call last):\n File \"\", line 11, in \n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/build.py\",\n line 83, in build_editable\n return os.path.basename(next(builder.build(directory=wheel_directory,\n versions=[\"editable\"])))\n \n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/plugin/interface.py\",\n line 157, in build\n artifact = version_api[version](directory, **build_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 547, in build_editable\n return self.build_editable_detection(directory, **build_data)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 559, in build_editable_detection\n for included_file in self.recurse_selected_project_files():\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/plugin/interface.py\",\n line 182, in recurse_selected_project_files\n if self.config.only_include:\n ^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/functools.py\", line 995, in __get__\n val = self.func(instance)\n ^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/config.py\",\n line 715, in only_include\n only_include = only_include_config.get(\"only-include\",\n self.default_only_include()) or self.packages\n \n ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 293, in default_only_include\n return self.default_file_selection_options.only_include\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/usr/lib/python3.12/functools.py\", line 995, in __get__\n val = self.func(instance)\n ^^^^^^^^^^^^^^^^^^^\n File\n \"/home/ibrahim/.cache/uv/builds-v0/.tmpWIdl99/lib/python3.12/site-packages/hatchling/builders/wheel.py\",\n line 281, in default_file_selection_options\n raise ValueError(message)\n ValueError: Unable to determine which files to ship\n inside the wheel using the following heuristics:\n https://hatch.pypa.io/latest/plugins/builder/wheel/#default-file-selection\n\n The most likely cause of this is that there is no directory that matches\n the name of your project (sb21_python_ingestion_api_prompt_low).\n\n At least one file selection option must be defined\n in the `tool.hatch.build.targets.wheel` table, see:\n https://hatch.pypa.io/latest/config/build/\n\n As an example, if you intend to ship a directory named `foo` that\n resides within a `src` directory located at the root of your project,\n you can define the following:\n\n [tool.hatch.build.targets.wheel]\n packages = [\"src/foo\"]\n\n\nhint: Build failures usually indicate a problem with the package or the build environment\n", - "startedAtMs": 1787852004352, - "lastActivityAtMs": 1787852006743 - }, - "build": { - "command": "compile not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "checkTypes": { - "command": "typecheck not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "sourceHash": "ad6bbc4f75569d509af60b4bdd3524070e9df40451ea654096c1f8bfff05d3b5", - "cacheKey": "be624c1574b3889b5fd15924d42bca7deac51e5bdbf88127a9ada299d32486ce", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 16, - "total": 16, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "build-failed", - "format-failed", - "install-failed", - "lint-failed", - "test-failed", - "typecheck-failed", - "validation-failed" - ], - "outcome": "model-failure" - }, - { - "id": "frontier-effect-eventsourcing-gemini-3.7-flash-low-prompt-r01", - "specId": "frontier-effect-eventsourcing", - "specTitle": "Frontier: TypeScript Effect service with event-sourcing/CQRS and tRPC-over-WebSocket subscriptions", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/frontier-effect-eventsourcing-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-frontier-effect-eventsourcing-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/frontier-effect-eventsourcing-gemini-3.7-flash-low-prompt-r01/sb21-frontier-effect-eventsourcing-prompt-low", - "codeMetrics": { - "files": 13, - "lines": 991, - "bytes": 32061 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 140036, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 509, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ @trpc/server@10.45.4\n+ effect@3.22.1\n+ ws@8.21.3\n+ zod@3.25.76\n+ @types/node@20.19.43\n+ @types/ws@8.18.1\n+ ts-node@10.9.2\n+ typescript@5.9.3\n\n28 packages installed [505.00ms]\n", - "stderrTail": "[0.31ms] migrated lockfile from package-lock.json\nSaved lockfile\n", - "startedAtMs": 1787852006773, - "lastActivityAtMs": 1787852007281 - }, - "build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 0, - "timedOut": false, - "durationMs": 2330, - "stdoutTail": "", - "stderrTail": "$ tsc\n", - "startedAtMs": 1787852007282, - "lastActivityAtMs": 1787852007285 - }, - "typecheck": { - "command": "/home/ibrahim/.bun/bin/bun run typecheck", - "exitCode": 0, - "timedOut": false, - "durationMs": 2005, - "stdoutTail": "", - "stderrTail": "$ tsc --noEmit\n", - "startedAtMs": 1787852009612, - "lastActivityAtMs": 1787852009614 - }, - "lint": { - "command": "lint (no linter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "format": { - "command": "format (no formatter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "test": { - "command": "/home/ibrahim/.bun/bin/bun run test", - "exitCode": null, - "timedOut": true, - "timeoutKind": "hard", - "durationMs": 1200069, - "stdoutTail": "--- Starting Effect Event-Sourcing Ledger Tests ---\n🚀 Bank Ledger tRPC WebSocket Server running on ws://localhost:3000\n1. Testing Account Opening...\nAccount opened event: {\n _tag: 'AccountOpened',\n id: 'evt-1787852015369-agkpu31',\n accountId: 'acc-1',\n owner: 'Alice',\n initialBalance: 100,\n timestamp: 1787852015369,\n sequence: 1\n}\n2. Testing Deposits...\nDeposits successful: 50 25\n3. Testing Withdrawals...\nWithdrawal successful: 75\n4. Testing Overdraft Rejection...\nOverdraft correctly rejected with InsufficientFundsError\n5. Verifying Read-Side Balance Projection...\nBalance view: {\n accountId: 'acc-1',\n owner: 'Alice',\n balance: 100,\n lastSequence: 4,\n transactionCount: 4,\n updatedAt: 1787852015371\n}\n6. Verifying Event Store Events...\n7. Testing Projection Rebuild from Zero...\nRebuilt balance view: {\n accountId: 'acc-1',\n owner: 'Alice',\n balance: 100,\n lastSequence: 4,\n transactionCount: 4,\n updatedAt: 1787852015371\n}\n8. Testing Idempotency (Re-applying same event)...\nProjection idempotency verified successfully!\n9. Testing Outbox Pattern...\nPending outbox entries: 0 (already processed during command dispatch)\n\n✅ All unit and domain integration tests PASSED!\n", - "stderrTail": "$ node --loader ts-node/esm test/index.ts\n(node:13608) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`:\n--import 'data:text/javascript,import { register } from \"node:module\"; import { pathToFileURL } from \"node:url\"; register(\"ts-node/esm\", pathToFileURL(\"./\"));'\n(Use `node --trace-warnings ...` to show where the warning was created)\n(node:13608) [DEP0180] DeprecationWarning: fs.Stats constructor is deprecated.\n(Use `node --trace-deprecation ...` to show where the warning was created)\nerror: script \"test\" was terminated by signal SIGTERM (Polite quit request)\n", - "startedAtMs": 1787852011617, - "lastActivityAtMs": 1787853211685 - } - }, - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 509, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ @trpc/server@10.45.4\n+ effect@3.22.1\n+ ws@8.21.3\n+ zod@3.25.76\n+ @types/node@20.19.43\n+ @types/ws@8.18.1\n+ ts-node@10.9.2\n+ typescript@5.9.3\n\n28 packages installed [505.00ms]\n", - "stderrTail": "[0.31ms] migrated lockfile from package-lock.json\nSaved lockfile\n", - "startedAtMs": 1787852006773, - "lastActivityAtMs": 1787852007281 - }, - "build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 0, - "timedOut": false, - "durationMs": 2330, - "stdoutTail": "", - "stderrTail": "$ tsc\n", - "startedAtMs": 1787852007282, - "lastActivityAtMs": 1787852007285 - }, - "checkTypes": { - "command": "/home/ibrahim/.bun/bin/bun run typecheck", - "exitCode": 0, - "timedOut": false, - "durationMs": 2005, - "stdoutTail": "", - "stderrTail": "$ tsc --noEmit\n", - "startedAtMs": 1787852009612, - "lastActivityAtMs": 1787852009614 - }, - "lint": { - "command": "lint (no linter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "format": { - "command": "format (no formatter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "test": { - "command": "/home/ibrahim/.bun/bin/bun run test", - "exitCode": null, - "timedOut": true, - "timeoutKind": "hard", - "durationMs": 1200069, - "stdoutTail": "--- Starting Effect Event-Sourcing Ledger Tests ---\n🚀 Bank Ledger tRPC WebSocket Server running on ws://localhost:3000\n1. Testing Account Opening...\nAccount opened event: {\n _tag: 'AccountOpened',\n id: 'evt-1787852015369-agkpu31',\n accountId: 'acc-1',\n owner: 'Alice',\n initialBalance: 100,\n timestamp: 1787852015369,\n sequence: 1\n}\n2. Testing Deposits...\nDeposits successful: 50 25\n3. Testing Withdrawals...\nWithdrawal successful: 75\n4. Testing Overdraft Rejection...\nOverdraft correctly rejected with InsufficientFundsError\n5. Verifying Read-Side Balance Projection...\nBalance view: {\n accountId: 'acc-1',\n owner: 'Alice',\n balance: 100,\n lastSequence: 4,\n transactionCount: 4,\n updatedAt: 1787852015371\n}\n6. Verifying Event Store Events...\n7. Testing Projection Rebuild from Zero...\nRebuilt balance view: {\n accountId: 'acc-1',\n owner: 'Alice',\n balance: 100,\n lastSequence: 4,\n transactionCount: 4,\n updatedAt: 1787852015371\n}\n8. Testing Idempotency (Re-applying same event)...\nProjection idempotency verified successfully!\n9. Testing Outbox Pattern...\nPending outbox entries: 0 (already processed during command dispatch)\n\n✅ All unit and domain integration tests PASSED!\n", - "stderrTail": "$ node --loader ts-node/esm test/index.ts\n(node:13608) ExperimentalWarning: `--experimental-loader` may be removed in the future; instead use `register()`:\n--import 'data:text/javascript,import { register } from \"node:module\"; import { pathToFileURL } from \"node:url\"; register(\"ts-node/esm\", pathToFileURL(\"./\"));'\n(Use `node --trace-warnings ...` to show where the warning was created)\n(node:13608) [DEP0180] DeprecationWarning: fs.Stats constructor is deprecated.\n(Use `node --trace-deprecation ...` to show where the warning was created)\nerror: script \"test\" was terminated by signal SIGTERM (Polite quit request)\n", - "startedAtMs": 1787852011617, - "lastActivityAtMs": 1787853211685 - }, - "sourceHash": "b1a63dd3e4f4d257e7e77ce07560316bdddc06efaa2fe6a6b79d04335c9e72c3", - "cacheKey": "d94b98426ba27c677605765a600cff5f89827e440aa5a0806066b12a3801703b", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 14, - "total": 14, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "format-failed", - "lint-failed", - "test-failed" - ], - "outcome": "success" - }, - { - "id": "react-native-expo-gemini-3.7-flash-low-prompt-r01", - "specId": "react-native-expo", - "specTitle": "React Native Expo habit tracker with Expo Router, Uniwind, MMKV, and Maestro + RNTL", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/react-native-expo-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-react-native-expo-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/react-native-expo-gemini-3.7-flash-low-prompt-r01/sb21-react-native-expo-prompt-low", - "codeMetrics": { - "files": 20, - "lines": 1500, - "bytes": 48535 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 258778, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 4962, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ expo@52.0.49\n+ expo-constants@17.0.8\n+ expo-linking@7.0.5\n+ expo-notifications@0.29.14\n+ expo-router@4.0.22\n+ expo-status-bar@2.0.1\n+ expo-updates@0.26.19\n+ react@18.3.1\n+ react-dom@18.3.1\n+ react-native@0.76.7\n+ react-native-mmkv@3.3.3\n+ react-native-safe-area-context@4.12.0\n+ react-native-screens@4.4.0\n+ tailwindcss@4.3.3\n+ uniwind@1.11.0\n+ @babel/core@7.29.7\n+ @testing-library/react-native@13.3.3\n+ @types/jest@29.5.14\n+ @types/react@18.3.31\n+ jest@29.7.0\n+ jest-expo@52.0.6\n+ react-test-renderer@18.3.1\n+ typescript@5.9.3\n\n970 packages installed [4.96s]\n", - "stderrTail": "[6.71ms] migrated lockfile from package-lock.json\nSaved lockfile\n", - "startedAtMs": 1787853211751, - "lastActivityAtMs": 1787853216710 - }, - "build": { - "command": "npx expo export --platform web", - "exitCode": 1, - "timedOut": false, - "durationMs": 1469, - "stdoutTail": "", - "stderrTail": "CommandError: It looks like you're trying to use web support but don't have the required\ndependencies installed.\n\nPlease install react-native-web@~0.19.13 by running:\n\nnpx expo install react-native-web\n\nIf you're not using web, please ensure you remove the \"web\" string from the\nplatforms array in the project Expo config.\n\n", - "startedAtMs": 1787853216713, - "lastActivityAtMs": 1787853218161 - }, - "not-run:typecheck": { - "command": "typecheck not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - } - }, - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 4962, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ expo@52.0.49\n+ expo-constants@17.0.8\n+ expo-linking@7.0.5\n+ expo-notifications@0.29.14\n+ expo-router@4.0.22\n+ expo-status-bar@2.0.1\n+ expo-updates@0.26.19\n+ react@18.3.1\n+ react-dom@18.3.1\n+ react-native@0.76.7\n+ react-native-mmkv@3.3.3\n+ react-native-safe-area-context@4.12.0\n+ react-native-screens@4.4.0\n+ tailwindcss@4.3.3\n+ uniwind@1.11.0\n+ @babel/core@7.29.7\n+ @testing-library/react-native@13.3.3\n+ @types/jest@29.5.14\n+ @types/react@18.3.31\n+ jest@29.7.0\n+ jest-expo@52.0.6\n+ react-test-renderer@18.3.1\n+ typescript@5.9.3\n\n970 packages installed [4.96s]\n", - "stderrTail": "[6.71ms] migrated lockfile from package-lock.json\nSaved lockfile\n", - "startedAtMs": 1787853211751, - "lastActivityAtMs": 1787853216710 - }, - "build": { - "command": "npx expo export --platform web", - "exitCode": 1, - "timedOut": false, - "durationMs": 1469, - "stdoutTail": "", - "stderrTail": "CommandError: It looks like you're trying to use web support but don't have the required\ndependencies installed.\n\nPlease install react-native-web@~0.19.13 by running:\n\nnpx expo install react-native-web\n\nIf you're not using web, please ensure you remove the \"web\" string from the\nplatforms array in the project Expo config.\n\n", - "startedAtMs": 1787853216713, - "lastActivityAtMs": 1787853218161 - }, - "checkTypes": { - "command": "typecheck not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "sourceHash": "a012172940acc618288992c36d90a639e868f3064fdda4fdacfeccabfd1449a1", - "cacheKey": "b75950880cad7a471b1a8b82ed472a8be38915f567c38cf14dc34075e0ffde3c", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 13, - "total": 13, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "build-failed", - "format-failed", - "lint-failed", - "test-failed", - "typecheck-failed", - "validation-failed" - ], - "outcome": "model-failure" - }, - { - "id": "go-realtime-api-gemini-3.7-flash-low-prompt-r01", - "specId": "go-realtime-api", - "specTitle": "Go realtime API with Chi, Ent, gRPC, NATS, Redis, and OpenTelemetry", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/go-realtime-api-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-go-realtime-api-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/go-realtime-api-gemini-3.7-flash-low-prompt-r01/sb21-go-realtime-api-prompt-low", - "codeMetrics": { - "files": 49, - "lines": 9261, - "bytes": 276814 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 136043, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "go mod download", - "exitCode": 0, - "timedOut": false, - "durationMs": 99, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853218509, - "lastActivityAtMs": 1787853218509 - }, - "build": { - "command": "go build ./...", - "exitCode": 0, - "timedOut": false, - "durationMs": 4165, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853218608, - "lastActivityAtMs": 1787853218608 - }, - "tidy": { - "command": "go mod tidy (advisory diff)", - "exitCode": 1, - "timedOut": false, - "durationMs": 614, - "stdoutTail": "od v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=\n golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=\n golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=\n-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=\n-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=\n-golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=\n-golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=\n-golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=\n-golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=\n-golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=\n-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\n-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=\n-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\n-golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=\n-golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=\n-golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=\n-golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=\n golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=\n golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=\n-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=\n-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=\n-golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=\n-golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=\n-golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=\n golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=\n golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=\n-golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=\n-golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=\n-golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=\n-golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=\n-golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=\n-golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE=\n-golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk=\n+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=\n+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=\n google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk=\n google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=\n google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU=\n google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8=\n-google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=\n-google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=\n google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=\n google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=\n gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\n+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=\n+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=\n gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\n gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n\n", - "stderrTail": "", - "startedAtMs": 1787853222773, - "lastActivityAtMs": 1787853223381 - }, - "lint": { - "command": "go vet ./...", - "exitCode": 0, - "timedOut": false, - "durationMs": 4078, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853223387, - "lastActivityAtMs": 1787853223387 - }, - "format": { - "command": "gofmt -l .", - "exitCode": 1, - "timedOut": false, - "durationMs": 57, - "stdoutTail": "internal/config/config.go\n", - "stderrTail": "", - "startedAtMs": 1787853227465, - "lastActivityAtMs": 1787853227499 - }, - "not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - } - }, - "install": { - "command": "go mod download", - "exitCode": 0, - "timedOut": false, - "durationMs": 99, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853218509, - "lastActivityAtMs": 1787853218509 - }, - "build": { - "command": "go build ./...", - "exitCode": 0, - "timedOut": false, - "durationMs": 4165, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853218608, - "lastActivityAtMs": 1787853218608 - }, - "lint": { - "command": "go vet ./...", - "exitCode": 0, - "timedOut": false, - "durationMs": 4078, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853223387, - "lastActivityAtMs": 1787853223387 - }, - "format": { - "command": "gofmt -l .", - "exitCode": 1, - "timedOut": false, - "durationMs": 57, - "stdoutTail": "internal/config/config.go\n", - "stderrTail": "", - "startedAtMs": 1787853227465, - "lastActivityAtMs": 1787853227499 - }, - "test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "sourceHash": "351917ff87b684c8fddead400d78c18007d63c1e44f8382ca046c5781ec54d24", - "cacheKey": "f48e84f9e750ee72f76b58c5e9f0637cdfc636e6ad7a899e08b7210bf85f6529", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 16, - "total": 16, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "format-failed", - "test-failed" - ], - "outcome": "success" - }, - { - "id": "java-spring-jooq-keycloak-gemini-3.7-flash-low-prompt-r01", - "specId": "java-spring-jooq-keycloak", - "specTitle": "Java Spring Boot API with jOOQ, Keycloak, GraphQL, and property/architecture tests", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/java-spring-jooq-keycloak-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-java-spring-jooq-keycloak-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/java-spring-jooq-keycloak-gemini-3.7-flash-low-prompt-r01/sb21-java-spring-jooq-keycloak-prompt-low", - "codeMetrics": { - "files": 23, - "lines": 1132, - "bytes": 37250 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 207073, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "build": { - "command": "mvn -q -B -DskipTests compile", - "exitCode": 0, - "timedOut": false, - "durationMs": 4398, - "stdoutTail": "", - "stderrTail": "Picked up JAVA_TOOL_OPTIONS: -XX:ActiveProcessorCount=2\n", - "startedAtMs": 1787853227577, - "lastActivityAtMs": 1787853227639 - }, - "test": { - "command": "mvn -q -B test", - "exitCode": 0, - "timedOut": false, - "durationMs": 17969, - "stdoutTail": "ustomizers' of type [org.springframework.boot.autoconfigure.transaction.TransactionManagerCustomizers] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). Is this bean getting eagerly injected into a currently created BeanPostProcessor [jobRegistryBeanPostProcessor]? Check the corresponding BeanPostProcessor declaration and its dependencies.\n2026-08-27 20:54:06.364 [main] WARN o.s.c.s.PostProcessorRegistrationDelegate$BeanPostProcessorChecker - Bean 'transactionManager' of type [org.springframework.jdbc.support.JdbcTransactionManager] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). Is this bean getting eagerly injected into a currently created BeanPostProcessor [jobRegistryBeanPostProcessor]? Check the corresponding BeanPostProcessor declaration and its dependencies.\n2026-08-27 20:54:06.369 [main] WARN o.s.c.s.PostProcessorRegistrationDelegate$BeanPostProcessorChecker - Bean 'spring.batch-org.springframework.boot.autoconfigure.batch.BatchProperties' of type [org.springframework.boot.autoconfigure.batch.BatchProperties] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). Is this bean getting eagerly injected into a currently created BeanPostProcessor [jobRegistryBeanPostProcessor]? Check the corresponding BeanPostProcessor declaration and its dependencies.\n2026-08-27 20:54:06.372 [main] WARN o.s.c.s.PostProcessorRegistrationDelegate$BeanPostProcessorChecker - Bean 'org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration$SpringBootBatchConfiguration' of type [org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration$SpringBootBatchConfiguration] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). The currently created BeanPostProcessor [jobRegistryBeanPostProcessor] is declared through a non-static factory method on that class; consider declaring it as static instead.\n2026-08-27 20:54:06.416 [main] INFO com.zaxxer.hikari.HikariDataSource - HikariPool-2 - Starting...\n2026-08-27 20:54:06.417 [main] INFO com.zaxxer.hikari.pool.HikariPool - HikariPool-2 - Added connection conn10: url=jdbc:h2:mem:testdb user=SA\n2026-08-27 20:54:06.417 [main] INFO com.zaxxer.hikari.HikariDataSource - HikariPool-2 - Start completed.\n2026-08-27 20:54:06.870 [main] INFO o.s.g.e.DefaultSchemaResourceGraphQlSourceBuilder - Loaded 1 resource(s) in the GraphQL schema.\n2026-08-27 20:54:06.879 [main] INFO o.s.b.a.g.GraphQlAutoConfiguration - GraphQL schema inspection:\n\tUnmapped fields: {}\n\tUnmapped registrations: {}\n\tUnmapped arguments: {}\n\tSkipped types: []\n2026-08-27 20:54:06.882 [main] INFO o.s.b.a.g.s.GraphQlWebMvcAutoConfiguration - GraphQL endpoint HTTP POST /graphql\n2026-08-27 20:54:06.955 [main] INFO o.s.b.a.e.web.EndpointLinksResolver - Exposing 3 endpoints beneath base path '/actuator'\n2026-08-27 20:54:07.304 [main] INFO c.e.demo.ItemGraphQLControllerTest - Started ItemGraphQLControllerTest in 1.359 seconds (process running for 11.639)\n2026-08-27 20:54:07.451 [main] INFO c.tngtech.archunit.core.PluginLoader - Detected Java version 21.0.12.1\ntimestamp = 2026-08-27T20:54:09.802162273, ItemPropertyTest:itemDtoPreservesValues = \n |-----------------------jqwik-----------------------\ntries = 1000 | # of calls to property\nchecks = 1000 | # of not rejected calls\ngeneration = RANDOMIZED | parameters are randomly generated\nafter-failure = SAMPLE_FIRST | try previously failed sample, then previous seed\nwhen-fixed-seed = ALLOW | fixing the random seed is allowed\nedge-cases#mode = MIXIN | edge cases are mixed in\nedge-cases#total = 48 | # of all combined edge cases\nedge-cases#tried = 48 | # of edge cases tried in current run\nseed = -7205641692078962707 | random seed to reproduce generated values\n\n\n", - "stderrTail": "Picked up JAVA_TOOL_OPTIONS: -XX:ActiveProcessorCount=2\nPicked up JAVA_TOOL_OPTIONS: -XX:ActiveProcessorCount=2\nAug 27, 2026 8:53:55 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:55 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nOpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader classes because bootstrap classpath has been appended\nWARNING: A Java agent has been loaded dynamically (/home/ibrahim/.m2/repository/net/bytebuddy/byte-buddy-agent/1.14.19/byte-buddy-agent-1.14.19.jar)\nWARNING: If a serviceability tool is in use, please run with -XX:+EnableDynamicAgentLoading to hide this warning\nWARNING: If a serviceability tool is not in use, please run with -Djdk.instrument.traceUsage for more information\nWARNING: Dynamic loading of agents will be disallowed by default in a future release\nSLF4J: No SLF4J providers were found.\nSLF4J: Defaulting to no-operation (NOP) logger implementation\nSLF4J: See https://www.slf4j.org/codes.html#noProviders for further details.\nSLF4J: Class path contains SLF4J bindings targeting slf4j-api versions 1.7.x or earlier.\nSLF4J: Ignoring binding found at [jar:file:/home/ibrahim/.m2/repository/org/wiremock/wiremock-standalone/3.0.1/wiremock-standalone-3.0.1.jar!/wiremock/org/slf4j/impl/StaticLoggerBinder.class]\nSLF4J: See https://www.slf4j.org/codes.html#ignoredBindings for an explanation.\n", - "startedAtMs": 1787853231975, - "lastActivityAtMs": 1787853249803 - } - }, - "build": { - "command": "mvn -q -B -DskipTests compile", - "exitCode": 0, - "timedOut": false, - "durationMs": 4398, - "stdoutTail": "", - "stderrTail": "Picked up JAVA_TOOL_OPTIONS: -XX:ActiveProcessorCount=2\n", - "startedAtMs": 1787853227577, - "lastActivityAtMs": 1787853227639 - }, - "test": { - "command": "mvn -q -B test", - "exitCode": 0, - "timedOut": false, - "durationMs": 17969, - "stdoutTail": "ustomizers' of type [org.springframework.boot.autoconfigure.transaction.TransactionManagerCustomizers] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). Is this bean getting eagerly injected into a currently created BeanPostProcessor [jobRegistryBeanPostProcessor]? Check the corresponding BeanPostProcessor declaration and its dependencies.\n2026-08-27 20:54:06.364 [main] WARN o.s.c.s.PostProcessorRegistrationDelegate$BeanPostProcessorChecker - Bean 'transactionManager' of type [org.springframework.jdbc.support.JdbcTransactionManager] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). Is this bean getting eagerly injected into a currently created BeanPostProcessor [jobRegistryBeanPostProcessor]? Check the corresponding BeanPostProcessor declaration and its dependencies.\n2026-08-27 20:54:06.369 [main] WARN o.s.c.s.PostProcessorRegistrationDelegate$BeanPostProcessorChecker - Bean 'spring.batch-org.springframework.boot.autoconfigure.batch.BatchProperties' of type [org.springframework.boot.autoconfigure.batch.BatchProperties] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). Is this bean getting eagerly injected into a currently created BeanPostProcessor [jobRegistryBeanPostProcessor]? Check the corresponding BeanPostProcessor declaration and its dependencies.\n2026-08-27 20:54:06.372 [main] WARN o.s.c.s.PostProcessorRegistrationDelegate$BeanPostProcessorChecker - Bean 'org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration$SpringBootBatchConfiguration' of type [org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration$SpringBootBatchConfiguration] is not eligible for getting processed by all BeanPostProcessors (for example: not eligible for auto-proxying). The currently created BeanPostProcessor [jobRegistryBeanPostProcessor] is declared through a non-static factory method on that class; consider declaring it as static instead.\n2026-08-27 20:54:06.416 [main] INFO com.zaxxer.hikari.HikariDataSource - HikariPool-2 - Starting...\n2026-08-27 20:54:06.417 [main] INFO com.zaxxer.hikari.pool.HikariPool - HikariPool-2 - Added connection conn10: url=jdbc:h2:mem:testdb user=SA\n2026-08-27 20:54:06.417 [main] INFO com.zaxxer.hikari.HikariDataSource - HikariPool-2 - Start completed.\n2026-08-27 20:54:06.870 [main] INFO o.s.g.e.DefaultSchemaResourceGraphQlSourceBuilder - Loaded 1 resource(s) in the GraphQL schema.\n2026-08-27 20:54:06.879 [main] INFO o.s.b.a.g.GraphQlAutoConfiguration - GraphQL schema inspection:\n\tUnmapped fields: {}\n\tUnmapped registrations: {}\n\tUnmapped arguments: {}\n\tSkipped types: []\n2026-08-27 20:54:06.882 [main] INFO o.s.b.a.g.s.GraphQlWebMvcAutoConfiguration - GraphQL endpoint HTTP POST /graphql\n2026-08-27 20:54:06.955 [main] INFO o.s.b.a.e.web.EndpointLinksResolver - Exposing 3 endpoints beneath base path '/actuator'\n2026-08-27 20:54:07.304 [main] INFO c.e.demo.ItemGraphQLControllerTest - Started ItemGraphQLControllerTest in 1.359 seconds (process running for 11.639)\n2026-08-27 20:54:07.451 [main] INFO c.tngtech.archunit.core.PluginLoader - Detected Java version 21.0.12.1\ntimestamp = 2026-08-27T20:54:09.802162273, ItemPropertyTest:itemDtoPreservesValues = \n |-----------------------jqwik-----------------------\ntries = 1000 | # of calls to property\nchecks = 1000 | # of not rejected calls\ngeneration = RANDOMIZED | parameters are randomly generated\nafter-failure = SAMPLE_FIRST | try previously failed sample, then previous seed\nwhen-fixed-seed = ALLOW | fixing the random seed is allowed\nedge-cases#mode = MIXIN | edge cases are mixed in\nedge-cases#total = 48 | # of all combined edge cases\nedge-cases#tried = 48 | # of edge cases tried in current run\nseed = -7205641692078962707 | random seed to reproduce generated values\n\n\n", - "stderrTail": "Picked up JAVA_TOOL_OPTIONS: -XX:ActiveProcessorCount=2\nPicked up JAVA_TOOL_OPTIONS: -XX:ActiveProcessorCount=2\nAug 27, 2026 8:53:55 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:55 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nAug 27, 2026 8:53:56 PM org.junit.platform.launcher.core.LauncherConfigurationParameters loadClasspathResource\nWARNING: Discovered 2 'junit-platform.properties' configuration files in the classpath; only the first will be used.\nOpenJDK 64-Bit Server VM warning: Sharing is only supported for boot loader classes because bootstrap classpath has been appended\nWARNING: A Java agent has been loaded dynamically (/home/ibrahim/.m2/repository/net/bytebuddy/byte-buddy-agent/1.14.19/byte-buddy-agent-1.14.19.jar)\nWARNING: If a serviceability tool is in use, please run with -XX:+EnableDynamicAgentLoading to hide this warning\nWARNING: If a serviceability tool is not in use, please run with -Djdk.instrument.traceUsage for more information\nWARNING: Dynamic loading of agents will be disallowed by default in a future release\nSLF4J: No SLF4J providers were found.\nSLF4J: Defaulting to no-operation (NOP) logger implementation\nSLF4J: See https://www.slf4j.org/codes.html#noProviders for further details.\nSLF4J: Class path contains SLF4J bindings targeting slf4j-api versions 1.7.x or earlier.\nSLF4J: Ignoring binding found at [jar:file:/home/ibrahim/.m2/repository/org/wiremock/wiremock-standalone/3.0.1/wiremock-standalone-3.0.1.jar!/wiremock/org/slf4j/impl/StaticLoggerBinder.class]\nSLF4J: See https://www.slf4j.org/codes.html#ignoredBindings for an explanation.\n", - "startedAtMs": 1787853231975, - "lastActivityAtMs": 1787853249803 - }, - "sourceHash": "c4b8f9ba21ec65b84b2b005f0f2a4492686f165bf92861eeb97de6d86defa8bc", - "cacheKey": "dcbead325845f3f3b27e61968b25e50370aec57972ab3c6d6aee28d4039af1af", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 28, - "total": 28, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [], - "outcome": "success" - }, - { - "id": "rust-leptos-axum-gemini-3.7-flash-low-prompt-r01", - "specId": "rust-leptos-axum", - "specTitle": "Rust Axum API with a Leptos WASM frontend and typed service libraries", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-rust-leptos-axum-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low", - "codeMetrics": { - "files": 15, - "lines": 1328, - "bytes": 46023 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 452864, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "cargoCheck": { - "command": "cargo check --workspace --all-targets", - "exitCode": 0, - "timedOut": false, - "durationMs": 137663, - "stdoutTail": "", - "stderrTail": "ash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/proto)\n Checking tcp-stream v0.28.0\n Checking leptos_config v0.6.15\n Checking tower v0.4.13\n Checking amq-protocol-uri v7.2.3\n Compiling rust-embed-utils v8.12.0\n Checking async-executor v1.14.0\n Checking async-lock v2.8.0\n Checking tokio-rustls v0.24.1\n Checking hyper-timeout v0.5.2\n Compiling askama_parser v0.2.1\n Compiling heck v0.4.1\n Compiling basic-toml v0.1.10\n Checking spin v0.9.9\n Checking socket2 v0.4.10\n Compiling crossbeam-epoch v0.9.20\n Checking colorchoice v1.0.5\n Compiling portable-atomic v1.15.0\n Checking anstyle-query v1.1.5\n Checking is_terminal_polyfill v1.70.2\n Checking anstyle v1.0.14\n Compiling amq-protocol v7.2.3\n Checking anstream v1.0.0\n Compiling askama_derive v0.12.5\n Checking flume v0.11.1\n Compiling sqlx-macros-core v0.7.4\n Checking tonic v0.12.3\n Checking hyper-rustls v0.24.2\n Checking async-global-executor v3.1.0\n Compiling rust-embed-impl v8.12.0\n Checking amq-protocol-tcp v7.2.3\n Checking humansize v2.1.3\n Compiling utoipa-swagger-ui v7.1.0\n Checking leptos v0.6.15\n Checking sb21-shared v0.1.0 (/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/shared)\n Checking tokio-rustls v0.26.4\n Checking matchers v0.2.0\n Checking webpki-roots v1.0.9\n Checking sharded-slab v0.1.7\n Checking reactor-trait v1.1.0\n Checking executor-trait v2.1.2\n Checking opentelemetry v0.23.0\n Checking ordered-float v4.6.0\n Checking tracing-log v0.2.0\n Checking tracing-serde v0.2.0\n Checking serde_test v1.0.177\n Checking thread_local v1.1.10\n Checking nu-ansi-term v0.50.3\n Checking glob v0.3.4\n Checking clap_lex v1.1.0\n Compiling lapin v2.5.5\n Checking sync_wrapper v0.1.2\n Checking askama_escape v0.10.3\n Checking reqwest v0.11.27\n Checking askama v0.12.1\n Checking clap_builder v4.6.6\n Checking opentelemetry_sdk v0.23.0\n Checking linear-map v1.2.0\n Checking tracing-subscriber v0.3.23\n Checking async-global-executor-trait v2.2.0\n Checking async-reactor-trait v1.1.0\n Checking hyper-rustls v0.27.9\n Checking rust-embed v8.12.0\n Checking pinky-swear v6.2.1\n Compiling sqlx-macros v0.7.4\n Checking tower-http v0.6.11\n Compiling clap_derive v4.6.4\n Checking tokio-test v0.4.5\n Checking crossbeam-channel v0.5.16\n Checking serde_qs v0.13.0\n Checking base64 v0.13.1\n Checking http-range-header v0.4.2\n Checking tagptr v0.2.0\n Checking moka v0.12.16\n Checking clap v4.6.6\n Checking tower-http v0.5.2\n Checking oauth2 v4.4.2\n Checking leptos_router v0.6.15\n Checking reqwest v0.12.28\n Checking sqlx v0.7.4\n Checking tracing-opentelemetry v0.24.0\n Checking askama_axum v0.4.0\n Checking leptos_meta v0.6.15\n Checking gloo-net v0.5.0\n Checking console_log v1.1.0\n Checking console_error_panic_hook v0.1.7\n Checking sb21-frontend v0.1.0 (/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/frontend)\n Checking sb21-server v0.1.0 (/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server)\n Finished `dev` profile [unoptimized + debuginfo] target(s) in 2m 17s\nwarning: the following packages contain code that will be rejected by a future version of Rust: proc-macro-error2 v2.0.1, sqlx-postgres v0.7.4\nnote: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1`\n", - "startedAtMs": 1787853345383, - "lastActivityAtMs": 1787853483003 - }, - "format": { - "command": "cargo fmt --check", - "exitCode": 1, - "timedOut": false, - "durationMs": 96, - "stdoutTail": "eptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:473:\n \n // Background Lapin Job worker mock / connector\n pub async fn start_background_jobs(amqp_url: String) {\n- info!(\"Attempting to connect background job worker to AMQP broker at {}\", amqp_url);\n- match lapin::Connection::connect(\n- &amqp_url,\n- lapin::ConnectionProperties::default(),\n- )\n- .await\n- {\n+ info!(\n+ \"Attempting to connect background job worker to AMQP broker at {}\",\n+ amqp_url\n+ );\n+ match lapin::Connection::connect(&amqp_url, lapin::ConnectionProperties::default()).await {\n Ok(_conn) => {\n info!(\"AMQP RabbitMQ connection established for job queue processor\");\n }\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:486:\n Err(err) => {\n- warn!(\"AMQP RabbitMQ not reachable ({}), running in standalone mode\", err);\n+ warn!(\n+ \"AMQP RabbitMQ not reachable ({}), running in standalone mode\",\n+ err\n+ );\n }\n }\n }\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:500:\n tracing_subscriber::registry()\n .with(tracing_subscriber::EnvFilter::new(\"info,sb21_server=debug\"))\n .with(tracing_subscriber::fmt::layer())\n- .with(tracing_opentelemetry::layer().with_tracer(tracer_provider.tracer(\"sb21-feature-flags\")))\n+ .with(\n+ tracing_opentelemetry::layer()\n+ .with_tracer(tracer_provider.tracer(\"sb21-feature-flags\")),\n+ )\n .init();\n \n info!(\"Starting Feature Flag Server with OpenTelemetry & Tracing initialized\");\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:509:\n ClientId::new(\"client_id\".to_string()),\n Some(ClientSecret::new(\"client_secret\".to_string())),\n AuthUrl::new(\"https://auth.example.com/oauth/authorize\".to_string())?,\n- Some(TokenUrl::new(\"https://auth.example.com/oauth/token\".to_string())?),\n+ Some(TokenUrl::new(\n+ \"https://auth.example.com/oauth/token\".to_string(),\n+ )?),\n );\n \n let cache: Cache = Cache::builder()\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:551:\n Some(pool)\n }\n Err(err) => {\n- warn!(\"PostgreSQL not reachable ({}), running with in-memory fallback\", err);\n+ warn!(\n+ \"PostgreSQL not reachable ({}), running with in-memory fallback\",\n+ err\n+ );\n None\n }\n };\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:574:\n tokio::spawn(async move {\n info!(\"gRPC Tonic server listening on {}\", grpc_addr);\n let svc = FeatureFlagServiceServer::new(GrpcFeatureFlagService { state: grpc_state });\n- if let Err(err) = TonicServer::builder().add_service(svc).serve(grpc_addr).await {\n+ if let Err(err) = TonicServer::builder()\n+ .add_service(svc)\n+ .serve(grpc_addr)\n+ .await\n+ {\n error!(\"gRPC server error: {}\", err);\n }\n });\n", - "stderrTail": "", - "startedAtMs": 1787853483047, - "lastActivityAtMs": 1787853483138 - }, - "not-run:lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - } - }, - "build": { - "command": "cargo check --workspace --all-targets", - "exitCode": 0, - "timedOut": false, - "durationMs": 137663, - "stdoutTail": "", - "stderrTail": "ash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/proto)\n Checking tcp-stream v0.28.0\n Checking leptos_config v0.6.15\n Checking tower v0.4.13\n Checking amq-protocol-uri v7.2.3\n Compiling rust-embed-utils v8.12.0\n Checking async-executor v1.14.0\n Checking async-lock v2.8.0\n Checking tokio-rustls v0.24.1\n Checking hyper-timeout v0.5.2\n Compiling askama_parser v0.2.1\n Compiling heck v0.4.1\n Compiling basic-toml v0.1.10\n Checking spin v0.9.9\n Checking socket2 v0.4.10\n Compiling crossbeam-epoch v0.9.20\n Checking colorchoice v1.0.5\n Compiling portable-atomic v1.15.0\n Checking anstyle-query v1.1.5\n Checking is_terminal_polyfill v1.70.2\n Checking anstyle v1.0.14\n Compiling amq-protocol v7.2.3\n Checking anstream v1.0.0\n Compiling askama_derive v0.12.5\n Checking flume v0.11.1\n Compiling sqlx-macros-core v0.7.4\n Checking tonic v0.12.3\n Checking hyper-rustls v0.24.2\n Checking async-global-executor v3.1.0\n Compiling rust-embed-impl v8.12.0\n Checking amq-protocol-tcp v7.2.3\n Checking humansize v2.1.3\n Compiling utoipa-swagger-ui v7.1.0\n Checking leptos v0.6.15\n Checking sb21-shared v0.1.0 (/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/shared)\n Checking tokio-rustls v0.26.4\n Checking matchers v0.2.0\n Checking webpki-roots v1.0.9\n Checking sharded-slab v0.1.7\n Checking reactor-trait v1.1.0\n Checking executor-trait v2.1.2\n Checking opentelemetry v0.23.0\n Checking ordered-float v4.6.0\n Checking tracing-log v0.2.0\n Checking tracing-serde v0.2.0\n Checking serde_test v1.0.177\n Checking thread_local v1.1.10\n Checking nu-ansi-term v0.50.3\n Checking glob v0.3.4\n Checking clap_lex v1.1.0\n Compiling lapin v2.5.5\n Checking sync_wrapper v0.1.2\n Checking askama_escape v0.10.3\n Checking reqwest v0.11.27\n Checking askama v0.12.1\n Checking clap_builder v4.6.6\n Checking opentelemetry_sdk v0.23.0\n Checking linear-map v1.2.0\n Checking tracing-subscriber v0.3.23\n Checking async-global-executor-trait v2.2.0\n Checking async-reactor-trait v1.1.0\n Checking hyper-rustls v0.27.9\n Checking rust-embed v8.12.0\n Checking pinky-swear v6.2.1\n Compiling sqlx-macros v0.7.4\n Checking tower-http v0.6.11\n Compiling clap_derive v4.6.4\n Checking tokio-test v0.4.5\n Checking crossbeam-channel v0.5.16\n Checking serde_qs v0.13.0\n Checking base64 v0.13.1\n Checking http-range-header v0.4.2\n Checking tagptr v0.2.0\n Checking moka v0.12.16\n Checking clap v4.6.6\n Checking tower-http v0.5.2\n Checking oauth2 v4.4.2\n Checking leptos_router v0.6.15\n Checking reqwest v0.12.28\n Checking sqlx v0.7.4\n Checking tracing-opentelemetry v0.24.0\n Checking askama_axum v0.4.0\n Checking leptos_meta v0.6.15\n Checking gloo-net v0.5.0\n Checking console_log v1.1.0\n Checking console_error_panic_hook v0.1.7\n Checking sb21-frontend v0.1.0 (/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/frontend)\n Checking sb21-server v0.1.0 (/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server)\n Finished `dev` profile [unoptimized + debuginfo] target(s) in 2m 17s\nwarning: the following packages contain code that will be rejected by a future version of Rust: proc-macro-error2 v2.0.1, sqlx-postgres v0.7.4\nnote: to see what the problems were, use the option `--future-incompat-report`, or run `cargo report future-incompatibilities --id 1`\n", - "startedAtMs": 1787853345383, - "lastActivityAtMs": 1787853483003 - }, - "lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "format": { - "command": "cargo fmt --check", - "exitCode": 1, - "timedOut": false, - "durationMs": 96, - "stdoutTail": "eptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:473:\n \n // Background Lapin Job worker mock / connector\n pub async fn start_background_jobs(amqp_url: String) {\n- info!(\"Attempting to connect background job worker to AMQP broker at {}\", amqp_url);\n- match lapin::Connection::connect(\n- &amqp_url,\n- lapin::ConnectionProperties::default(),\n- )\n- .await\n- {\n+ info!(\n+ \"Attempting to connect background job worker to AMQP broker at {}\",\n+ amqp_url\n+ );\n+ match lapin::Connection::connect(&amqp_url, lapin::ConnectionProperties::default()).await {\n Ok(_conn) => {\n info!(\"AMQP RabbitMQ connection established for job queue processor\");\n }\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:486:\n Err(err) => {\n- warn!(\"AMQP RabbitMQ not reachable ({}), running in standalone mode\", err);\n+ warn!(\n+ \"AMQP RabbitMQ not reachable ({}), running in standalone mode\",\n+ err\n+ );\n }\n }\n }\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:500:\n tracing_subscriber::registry()\n .with(tracing_subscriber::EnvFilter::new(\"info,sb21_server=debug\"))\n .with(tracing_subscriber::fmt::layer())\n- .with(tracing_opentelemetry::layer().with_tracer(tracer_provider.tracer(\"sb21-feature-flags\")))\n+ .with(\n+ tracing_opentelemetry::layer()\n+ .with_tracer(tracer_provider.tracer(\"sb21-feature-flags\")),\n+ )\n .init();\n \n info!(\"Starting Feature Flag Server with OpenTelemetry & Tracing initialized\");\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:509:\n ClientId::new(\"client_id\".to_string()),\n Some(ClientSecret::new(\"client_secret\".to_string())),\n AuthUrl::new(\"https://auth.example.com/oauth/authorize\".to_string())?,\n- Some(TokenUrl::new(\"https://auth.example.com/oauth/token\".to_string())?),\n+ Some(TokenUrl::new(\n+ \"https://auth.example.com/oauth/token\".to_string(),\n+ )?),\n );\n \n let cache: Cache = Cache::builder()\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:551:\n Some(pool)\n }\n Err(err) => {\n- warn!(\"PostgreSQL not reachable ({}), running with in-memory fallback\", err);\n+ warn!(\n+ \"PostgreSQL not reachable ({}), running with in-memory fallback\",\n+ err\n+ );\n None\n }\n };\nDiff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/rust-leptos-axum-gemini-3.7-flash-low-prompt-r01/sb21-rust-leptos-axum-prompt-low.validate-tmp/crates/server/src/main.rs:574:\n tokio::spawn(async move {\n info!(\"gRPC Tonic server listening on {}\", grpc_addr);\n let svc = FeatureFlagServiceServer::new(GrpcFeatureFlagService { state: grpc_state });\n- if let Err(err) = TonicServer::builder().add_service(svc).serve(grpc_addr).await {\n+ if let Err(err) = TonicServer::builder()\n+ .add_service(svc)\n+ .serve(grpc_addr)\n+ .await\n+ {\n error!(\"gRPC server error: {}\", err);\n }\n });\n", - "stderrTail": "", - "startedAtMs": 1787853483047, - "lastActivityAtMs": 1787853483138 - }, - "test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "sourceHash": "ac50b5e91e637eab28ff3dabf0cb44cef46d4032c1fc3bff284015a49dd72cea", - "cacheKey": "9d9f0474793b8c9f4ce88e4c8cd96ef7fb20db9a9f7b0463df386080819e360f", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 24, - "total": 24, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "format-failed", - "lint-failed", - "test-failed" - ], - "outcome": "success" - }, - { - "id": "multi-ts-go-grpc-gemini-3.7-flash-low-prompt-r01", - "specId": "multi-ts-go-grpc", - "specTitle": "Multi-ecosystem app: Nuxt (Vue) frontend with a Go Chi + gRPC backend", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-ts-go-grpc-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-multi-ts-go-grpc-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-ts-go-grpc-gemini-3.7-flash-low-prompt-r01/sb21-multi-ts-go-grpc-prompt-low", - "codeMetrics": { - "files": 77, - "lines": 9804, - "bytes": 665548 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 233437, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "backend:install": { - "command": "go mod download", - "exitCode": 1, - "timedOut": false, - "durationMs": 19, - "stdoutTail": "", - "stderrTail": "go: errors parsing go.mod:\ngo.mod:1: unexpected input character '\\x00'\n", - "startedAtMs": 1787853283953, - "lastActivityAtMs": 1787853283972 - }, - "backend:not-run:build": { - "command": "build not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "backend:not-run:tidy": { - "command": "tidy not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "backend:not-run:lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "backend:not-run:format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "backend:not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "unvalidated:no-build-surface": { - "command": "no build or typecheck surface was discovered. A green install alone is not a pass", - "exitCode": 1, - "timedOut": false, - "status": "ran", - "durationMs": 0, - "stdoutTail": "", - "stderrTail": "no build or typecheck surface was discovered. A green install alone is not a pass" - } - }, - "install": { - "command": "go mod download", - "exitCode": 1, - "timedOut": false, - "durationMs": 19, - "stdoutTail": "", - "stderrTail": "go: errors parsing go.mod:\ngo.mod:1: unexpected input character '\\x00'\n", - "startedAtMs": 1787853283953, - "lastActivityAtMs": 1787853283972 - }, - "build": { - "command": "build not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "sourceHash": "54a1b26df640f4b9f1f7ae202bcf6c65e44053838849849c80351c039e5fe4d0", - "cacheKey": "da07f01ef8b9e08586ad93e4be5f634e6f76198746d775f308bb1d64a4afaa78", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 20, - "total": 22, - "percent": 91, - "misses": [ - "frontend:tailwind", - "testing:testify+gomock" - ] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "build-failed", - "format-failed", - "install-failed", - "lint-failed", - "stack-mismatch", - "test-failed", - "validation-failed" - ], - "outcome": "model-failure" - }, - { - "id": "ai-search-workbench-gemini-3.7-flash-low-prompt-r01", - "specId": "ai-search-workbench", - "specTitle": "AI search workbench with split semantic/full-text search on the Vite+ toolchain", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/ai-search-workbench-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-ai-search-workbench-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/ai-search-workbench-gemini-3.7-flash-low-prompt-r01/sb21-ai-search-workbench-prompt-low", - "codeMetrics": { - "files": 50, - "lines": 1796, - "bytes": 55058 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 90681, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 1, - "timedOut": false, - "durationMs": 23184, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n", - "stderrTail": "Resolving dependencies\nResolved, downloaded and extracted [1291]\nerror: GET https://registry.npmjs.org/@opensearchproject%2fopensearch - 404\n\nerror: No version matching \"^0.62.0\" found for specifier \"@orpc/server\" (but package exists)\nerror: @orpc/server@^0.62.0 failed to resolve\nerror: @opensearchproject/opensearch@^3.1.0 failed to resolve\n", - "startedAtMs": 1787853249976, - "lastActivityAtMs": 1787853273150 - }, - "not-run:build": { - "command": "build not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:typecheck": { - "command": "typecheck not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "unvalidated:no-build-surface": { - "command": "no build or typecheck surface was discovered. A green install alone is not a pass", - "exitCode": 1, - "timedOut": false, - "status": "ran", - "durationMs": 0, - "stdoutTail": "", - "stderrTail": "no build or typecheck surface was discovered. A green install alone is not a pass" - } - }, - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 1, - "timedOut": false, - "durationMs": 23184, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n", - "stderrTail": "Resolving dependencies\nResolved, downloaded and extracted [1291]\nerror: GET https://registry.npmjs.org/@opensearchproject%2fopensearch - 404\n\nerror: No version matching \"^0.62.0\" found for specifier \"@orpc/server\" (but package exists)\nerror: @orpc/server@^0.62.0 failed to resolve\nerror: @opensearchproject/opensearch@^3.1.0 failed to resolve\n", - "startedAtMs": 1787853249976, - "lastActivityAtMs": 1787853273150 - }, - "build": { - "command": "build not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "checkTypes": { - "command": "typecheck not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "sourceHash": "7bac9abe70e307bcd56668b1dd50c99963a9a5640d9791f1c4fe0d0eac004086", - "cacheKey": "a44e4acddf2dc969e358341a0a744b6dbd6cf893d9564c3dc181006f10f0269a", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 23, - "total": 27, - "percent": 85, - "misses": [ - "ui:shadcn", - "search:opensearch", - "i18n:paraglide", - "addon:vite-plus" - ] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "build-failed", - "format-failed", - "install-failed", - "lint-failed", - "stack-mismatch", - "test-failed", - "typecheck-failed", - "validation-failed" - ], - "outcome": "model-failure" - }, - { - "id": "frontier-polyglot-proto-gemini-3.7-flash-low-prompt-r01", - "specId": "frontier-polyglot-proto", - "specTitle": "Frontier: polyglot monorepo, shared protobuf across a Rust gRPC service, a Go gateway, and a TS client", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/frontier-polyglot-proto-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-frontier-polyglot-proto-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/frontier-polyglot-proto-gemini-3.7-flash-low-prompt-r01/sb21-frontier-polyglot-proto-prompt-low", - "codeMetrics": { - "files": 14, - "lines": 2023, - "bytes": 60613 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 233198, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "prerequisite:01:buf": { - "command": "buf (no buf.gen.yaml / buf.gen.yml / buf.gen.json in the project)", - "exitCode": null, - "timedOut": false, - "status": "na", - "durationMs": 0, - "stdoutTail": "n/a", - "stderrTail": "" - }, - "client-ts:install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 933, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ @types/node@22.20.1\n+ ts-proto@2.12.1\n+ typescript@5.9.3\n+ @bufbuild/protobuf@2.14.0\n+ long@5.3.2\n+ protobufjs@7.6.6\n\n21 packages installed [929.00ms]\n\nBlocked 1 postinstall. Run `bun pm untrusted` for details.\n", - "stderrTail": "[0.23ms] migrated lockfile from package-lock.json\nResolving dependencies\nResolved, downloaded and extracted [8]\nSaved lockfile\n", - "startedAtMs": 1787853483400, - "lastActivityAtMs": 1787853484331 - }, - "client-ts:build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 0, - "timedOut": false, - "durationMs": 1581, - "stdoutTail": "", - "stderrTail": "$ bun run codegen && tsc\n$ protoc --plugin=protoc-gen-ts_proto=./node_modules/.bin/protoc-gen-ts_proto --ts_proto_out=src/generated --ts_proto_opt=esModuleInterop=true,outputServices=generic-definitions,env=browser,importSuffix=.js --proto_path=../proto ../proto/todo/v1/todo.proto\n", - "startedAtMs": 1787853484333, - "lastActivityAtMs": 1787853484339 - }, - "client-ts:typecheck": { - "command": "/home/ibrahim/.bun/bin/bunx tsc --build", - "exitCode": 0, - "timedOut": false, - "durationMs": 1383, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853485914, - "lastActivityAtMs": 1787853485914 - }, - "client-ts:lint": { - "command": "lint (no linter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "client-ts:format": { - "command": "format (no formatter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "client-ts:test": { - "command": "test (no test script)", - "exitCode": null, - "timedOut": false, - "status": "na", - "durationMs": 0, - "stdoutTail": "n/a", - "stderrTail": "" - }, - "services/core-rust:cargoCheck": { - "command": "cargo check --workspace --all-targets", - "exitCode": 0, - "timedOut": false, - "durationMs": 26813, - "stdoutTail": "", - "stderrTail": " Compiling proc-macro2 v1.0.107\n Compiling unicode-ident v1.0.24\n Compiling quote v1.0.47\n Compiling libc v0.2.189\n Compiling syn v2.0.119\n Checking pin-project-lite v0.2.17\n Checking futures-core v0.3.34\n Checking bytes v1.12.1\n Compiling syn v3.0.4\n Compiling anyhow v1.0.104\n Checking itoa v1.0.18\n Compiling either v1.18.0\n Checking slab v0.4.12\n Compiling itertools v0.14.0\n Checking http v1.5.0\n Compiling tokio-macros v2.7.2\n Checking socket2 v0.6.5\n Checking mio v1.2.2\n Compiling getrandom v0.4.3\n Checking tokio v1.53.1\n Compiling prost-derive v0.13.5\n Checking futures-task v0.3.34\n Checking cfg-if v1.0.4\n Checking once_cell v1.21.4\n Checking tracing-core v0.1.36\n Checking futures-util v0.3.34\n Checking http-body v1.1.0\n Compiling tracing-attributes v0.1.31\n Compiling zerocopy v0.8.56\n Checking futures-sink v0.3.34\n Compiling rustix v1.1.4\n Checking tower-service v0.3.3\n Checking tokio-util v0.7.19\n Checking tracing v0.1.44\n Checking equivalent v1.0.2\n Compiling linux-raw-sys v0.12.1\n Compiling regex-syntax v0.8.11\n Compiling hashbrown v0.17.1\n Compiling bitflags v2.13.1\n Compiling prettyplease v0.2.37\n Compiling httparse v1.10.1\n Checking indexmap v2.14.0\n Compiling regex-automata v0.4.18\n Compiling prost v0.13.5\n Checking getrandom v0.2.17\n Compiling fastrand v2.5.0\n Compiling serde_core v1.0.229\n Checking fnv v1.0.7\n Compiling autocfg v1.5.1\n Checking tower-layer v0.3.3\n Checking atomic-waker v1.1.2\n Compiling fixedbitset v0.5.7\n Checking try-lock v0.2.5\n Compiling rustversion v1.0.23\n Checking want v0.3.1\n Compiling tempfile v3.27.0\n Compiling petgraph v0.7.1\n Checking ppv-lite86 v0.2.21\n Checking h2 v0.4.19\n Compiling indexmap v1.9.3\n Checking rand_core v0.6.4\n Compiling prost-types v0.13.5\n Compiling regex v1.13.1\n Checking futures-channel v0.3.34\n Checking sync_wrapper v1.0.2\n Checking httpdate v1.0.3\n Compiling heck v0.5.0\n Checking smallvec v1.15.2\n Compiling serde v1.0.229\n Compiling multimap v0.10.1\n Compiling log v0.4.34\n Checking hyper v1.11.0\n Compiling prost-build v0.13.5\n Checking rand_chacha v0.3.1\n Checking http-body-util v0.1.5\n Compiling async-trait v0.1.92\n Compiling pin-project-internal v1.1.13\n Checking hashbrown v0.12.3\n Checking mime v0.3.17\n Checking axum-core v0.4.5\n Checking pin-project v1.1.13\n Checking rand v0.8.8\n Checking hyper-util v0.1.20\n Compiling tonic-build v0.12.3\n Checking tower v0.5.3\n Compiling async-stream-impl v0.3.6\n Checking percent-encoding v2.3.2\n Checking matchit v0.7.3\n Checking memchr v2.8.3\n Checking async-stream v0.3.6\n Compiling core-rust v0.1.0 (/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/frontier-polyglot-proto-gemini-3.7-flash-low-prompt-r01/sb21-frontier-polyglot-proto-prompt-low.validate-tmp/services/core-rust)\n Checking axum v0.7.9\n Checking hyper-timeout v0.5.2\n Checking tower v0.4.13\n Checking tokio-stream v0.1.19\n Checking socket2 v0.5.10\n Checking base64 v0.22.1\n Checking uuid v1.26.0\n Checking tonic v0.12.3\n Finished `dev` profile [unoptimized + debuginfo] target(s) in 26.79s\n", - "startedAtMs": 1787853487299, - "lastActivityAtMs": 1787853514100 - }, - "services/core-rust:format": { - "command": "cargo fmt --check", - "exitCode": 1, - "timedOut": false, - "durationMs": 49, - "stdoutTail": "Diff in /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/frontier-polyglot-proto-gemini-3.7-flash-low-prompt-r01/sb21-frontier-polyglot-proto-prompt-low.validate-tmp/services/core-rust/src/main.rs:53:\n Some(item) => Ok(Response::new(GetTodoResponse {\n item: Some(item.clone()),\n })),\n- None => Err(Status::not_found(format!(\"Todo with ID {} not found\", req.id))),\n+ None => Err(Status::not_found(format!(\n+ \"Todo with ID {} not found\",\n+ req.id\n+ ))),\n }\n }\n \n", - "stderrTail": "", - "startedAtMs": 1787853514112, - "lastActivityAtMs": 1787853514159 - }, - "services/core-rust:not-run:lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "services/core-rust:not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "gateway-go:install": { - "command": "go mod download", - "exitCode": 0, - "timedOut": false, - "durationMs": 20, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853514163, - "lastActivityAtMs": 1787853514163 - }, - "gateway-go:build": { - "command": "go build ./...", - "exitCode": 0, - "timedOut": false, - "durationMs": 896, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853514183, - "lastActivityAtMs": 1787853514183 - }, - "gateway-go:tidy": { - "command": "go mod tidy (advisory diff)", - "exitCode": 0, - "timedOut": false, - "durationMs": 51, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853515080, - "lastActivityAtMs": 1787853515080 - }, - "gateway-go:lint": { - "command": "go vet ./...", - "exitCode": 0, - "timedOut": false, - "durationMs": 1364, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853515131, - "lastActivityAtMs": 1787853515131 - }, - "gateway-go:format": { - "command": "gofmt -l .", - "exitCode": 0, - "timedOut": false, - "durationMs": 18, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853516495, - "lastActivityAtMs": 1787853516495 - }, - "gateway-go:test": { - "command": "go test ./...", - "exitCode": 0, - "timedOut": false, - "durationMs": 1416, - "stdoutTail": "? \tgateway-go\t[no test files]\n? \tgateway-go/proto/todo/v1\t[no test files]\n", - "stderrTail": "", - "startedAtMs": 1787853516514, - "lastActivityAtMs": 1787853517916 - } - }, - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 933, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ @types/node@22.20.1\n+ ts-proto@2.12.1\n+ typescript@5.9.3\n+ @bufbuild/protobuf@2.14.0\n+ long@5.3.2\n+ protobufjs@7.6.6\n\n21 packages installed [929.00ms]\n\nBlocked 1 postinstall. Run `bun pm untrusted` for details.\n", - "stderrTail": "[0.23ms] migrated lockfile from package-lock.json\nResolving dependencies\nResolved, downloaded and extracted [8]\nSaved lockfile\n", - "startedAtMs": 1787853483400, - "lastActivityAtMs": 1787853484331 - }, - "build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 0, - "timedOut": false, - "durationMs": 1581, - "stdoutTail": "", - "stderrTail": "$ bun run codegen && tsc\n$ protoc --plugin=protoc-gen-ts_proto=./node_modules/.bin/protoc-gen-ts_proto --ts_proto_out=src/generated --ts_proto_opt=esModuleInterop=true,outputServices=generic-definitions,env=browser,importSuffix=.js --proto_path=../proto ../proto/todo/v1/todo.proto\n", - "startedAtMs": 1787853484333, - "lastActivityAtMs": 1787853484339 - }, - "checkTypes": { - "command": "/home/ibrahim/.bun/bin/bunx tsc --build", - "exitCode": 0, - "timedOut": false, - "durationMs": 1383, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853485914, - "lastActivityAtMs": 1787853485914 - }, - "lint": { - "command": "lint (no linter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "format": { - "command": "format (no formatter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "test": { - "command": "test (no test script)", - "exitCode": null, - "timedOut": false, - "status": "na", - "durationMs": 0, - "stdoutTail": "n/a", - "stderrTail": "" - }, - "sourceHash": "9fc2ee0323542ec6b45aba2bd78126f314c3026c8e3d05aec5a21137a89ac7ed", - "cacheKey": "bb6c1af1bc7aa126605f99e7c048e6519c7914aa22b8563f5fb48c1d5733b361", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 10, - "total": 10, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "format-failed", - "lint-failed", - "test-failed" - ], - "outcome": "success" - }, - { - "id": "multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01", - "specId": "multi-dotnet-ops", - "specTitle": "Multi-ecosystem ops portal with TypeScript frontend and .NET Minimal API backend", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-multi-dotnet-ops-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low", - "codeMetrics": { - "files": 37, - "lines": 2162, - "bytes": 75179 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 222737, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 6468, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ @biomejs/biome@1.9.4 (v2.5.10 available)\n+ turbo@2.10.12\n\n133 packages installed [6.46s]\n", - "stderrTail": "moved pnpm-workspace.yaml to workspaces in package.json\n[1.85s] migrated lockfile from pnpm-lock.yaml\nSaved lockfile\n", - "startedAtMs": 1787853284002, - "lastActivityAtMs": 1787853290461 - }, - "build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 0, - "timedOut": false, - "durationMs": 18953, - "stdoutTail": "\n • Packages in scope: web\n • Running build in 1 packages\n • Remote caching disabled\n\nweb:build: cache miss, executing 11eac617d2802b8b\nweb:build: \nweb:build: > web@0.1.0 build /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/web\nweb:build: > next build\nweb:build: \nweb:build: ▲ Next.js 14.2.35\nweb:build: \nweb:build: Creating an optimized production build ...\nweb:build: ✓ Compiled successfully\nweb:build: Linting and checking validity of types ...\nweb:build: Collecting page data ...\nweb:build: Generating static pages (0/4) ...\nweb:build: Generating static pages (1/4) \r\nweb:build: Generating static pages (2/4) \r\nweb:build: Generating static pages (3/4) \r\nweb:build: ✓ Generating static pages (4/4)\nweb:build: Finalizing page optimization ...\nweb:build: Collecting build traces ...\nweb:build: \nweb:build: Route (app) Size First Load JS\nweb:build: ┌ ○ / 27.1 kB 114 kB\nweb:build: └ ○ /_not-found 871 B 88.1 kB\nweb:build: + First Load JS shared by all 87.3 kB\nweb:build: ├ chunks/183-271d519d1e0b7743.js 31.7 kB\nweb:build: ├ chunks/3cc391f0-ea67247d5ede1772.js 53.6 kB\nweb:build: └ other shared chunks (total) 1.92 kB\nweb:build: \nweb:build: \nweb:build: ○ (Static) prerendered as static content\nweb:build: \n\n Tasks: 1 successful, 1 total\nCached: 0 cached, 1 total\n Time: 18.857s \n\n", - "stderrTail": "$ turbo run build\n• turbo 2.10.12\n", - "startedAtMs": 1787853290470, - "lastActivityAtMs": 1787853309402 - }, - "lint": { - "command": "/home/ibrahim/.bun/bin/bun run lint", - "exitCode": 1, - "timedOut": false, - "durationMs": 236, - "stdoutTail": "Checked 21 files in 59ms. No fixes applied.\nFound 3 errors.\n", - "stderrTail": "/page.ts\":·{\n 70 │ + ······\"size\":·3653,\n 71 │ + ······\"mtime_nanos\":·1787853293691384141,\n 72 │ + ······\"mode\":·436,\n 73 │ + ······\"is_dir\":·false\n 74 │ + ····},\n 75 │ + ····\"apps/web/.next/server/app/_not-found\":·{\n 76 │ + ······\"size\":·0,\n 77 │ + ······\"mtime_nanos\":·0,\n 78 │ + ······\"mode\":·0,\n 79 │ + ······\"is_dir\":·true\n 80 │ + ····},\n 81 │ + ····\"apps/web/.next/server/chunks/font-manifest.json\":·{\n 82 │ + ······\"size\":·2,\n 83 │ + ······\"mtime_nanos\":·1787853293689384118,\n 84 │ + ······\"mode\":·436,\n 85 │ + ······\"is_dir\":·false\n 86 │ + ····},\n 87 │ + ····\"apps/web/.next/static/chunks/pages\":·{\n 88 │ + ······\"size\":·0,\n 89 │ + ······\"mtime_nanos\":·0,\n 90 │ + ······\"mode\":·0,\n 91 │ + ······\"is_dir\":·true\n 92 │ + ····},\n 93 │ + ····\"apps/web/.next/server/server-reference-manifest.json\":·{\n 94 │ + ······\"size\":·84,\n 95 │ + ······\"mtime_nanos\":·1787853293691384141,\n 96 │ + ······\"mode\":·436,\n 97 │ + ······\"is_dir\":·false\n 98 │ + ····},\n 99 │ + ····\"apps/web/.next/server/app/_not-found.meta\":·{\n 100 │ + ······\"size\":·142,\n 101 │ + ······\"mtime_nanos\":·1787853303888165031,\n 102 │ + ······\"mode\":·436,\n 103 │ + ······\"is_dir\":·false\n 104 │ + ····},\n 105 │ + ····\"apps/web/.next/server/app/page.js.nft.json\":·{\n 106 │ + ······\"size\":·1910,\n 107 │ + ······\"mtime_nanos\":·1787853309317556488,\n 108 │ + ······\"mode\":·436,\n 109 │ + ······\"is_dir\":·false\n 110 │ + ····},\n 111 │ + ····\"apps/web/.next/server/app/_not-found/page.js.nft.json\":·{\n 112 │ + ······\"size\":·1907,\n 113 │ + ······\"mtime_nanos\":·1787853309317556488,\n 114 │ + ······\"mode\":·436,\n 115 │ + ······\"is_dir\":·false\n 116 │ + ····},\n 117 │ + ····\"apps/web/.next/images-manifest.json\":·{\n 118 │ + ······\"size\":·511,\n 119 │ + ······\"mtime_nanos\":·1787853304118499280,\n 120 │ + ······\"mode\":·436,\n 121 │ + ······\"is_dir\":·false\n 122 │ + ····},\n 123 │ + ····\"apps/web/.next/server/app/page_client-reference-manifest.js\":·{\n 124 │ + ······\"size\":·6451,\n 125 │ + ······\"mtime_nanos\":·1787853299034443208,\n 126 │ + ······\"mode\":·436,\n 127 │ + ······\"is_dir\":·false\n 128 │ + ····},\n 129 │ + ····\"apps/web/.next/server/middleware-react-loadable-manifest.js\":·{\n 130 │ + ······\"size\":·36,\n 131 │ + ······\"mtime_nanos\":·1787853299032443186,\n 132 │ + ······\"mode\":·436,\n 133 │ + ······\"is_dir\":·false\n 134 │ + ····},\n 135 │ + ····\"apps/web/.next/static/chunks/main-app-2ad5ecf306059106.js\":·{\n 136 │ + ······\"size\":·460,\n 137 │ + ······\"mtime_nanos\":·1787853299033443197,\n 138 │ + ······\"mode\":·436,\n 139 │ + ······\"is_dir\":·false\n 140 │ + ····},\n 141 │ + ····\"apps/web/.next/server/functions-config-manifest.json\":·{\n 142 │ + ······\"size\":·28,\n 143 │ + ······\"mtime_nanos\":·1787853303542492934,\n 144 │ + ······\"mode\":·436,\n 145 │ + ······\"is_dir\":·false\n 146 │ + ····},\n 147 │ + ····\"apps/web/.next/static/chunks\":·{\n 148 │ + ······\"size\":·0,\n 149 │ + ······\"mtime_nanos\":·0,\n 453 more lines truncated\n \n\n./.turbo/cache/11eac617d2802b8b-meta.json format ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n × Formatter would have printed the following content:\n \n 1 │ - {\"hash\":\"11eac617d2802b8b\",\"duration\":18843,\"sha\":\"38ccd8d1d2dcedf7bb5738716af96f7d3142e618\",\"dirty_hash\":null}\n 1 │ + {\n 2 │ + ··\"hash\":·\"11eac617d2802b8b\",\n 3 │ + ··\"duration\":·18843,\n 4 │ + ··\"sha\":·\"38ccd8d1d2dcedf7bb5738716af96f7d3142e618\",\n 5 │ + ··\"dirty_hash\":·null\n 6 │ + }\n 7 │ + \n \n\ncheck ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n × Some errors were emitted while running checks.\n \n\nerror: script \"lint\" exited with code 1\n", - "startedAtMs": 1787853309423, - "lastActivityAtMs": 1787853309658 - }, - "not-run:format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "apps/web:install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 7, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\nChecked 135 installs across 154 packages (no changes) [4.00ms]\n", - "stderrTail": "", - "startedAtMs": 1787853309659, - "lastActivityAtMs": 1787853309665 - }, - "apps/web:build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 0, - "timedOut": false, - "durationMs": 13462, - "stdoutTail": " ▲ Next.js 14.2.35\n\n Creating an optimized production build ...\n ✓ Compiled successfully\n Linting and checking validity of types ...\n Collecting page data ...\n Generating static pages (0/4) ...\n Generating static pages (1/4) \r\n Generating static pages (2/4) \r\n Generating static pages (3/4) \r\n ✓ Generating static pages (4/4)\n Finalizing page optimization ...\n Collecting build traces ...\n\nRoute (app) Size First Load JS\n┌ ○ / 27.1 kB 114 kB\n└ ○ /_not-found 871 B 88.1 kB\n+ First Load JS shared by all 87.3 kB\n ├ chunks/183-271d519d1e0b7743.js 31.7 kB\n ├ chunks/3cc391f0-ea67247d5ede1772.js 53.6 kB\n └ other shared chunks (total) 1.92 kB\n\n\n○ (Static) prerendered as static content\n\n", - "stderrTail": "$ next build\n", - "startedAtMs": 1787853309666, - "lastActivityAtMs": 1787853323086 - }, - "apps/web:typecheck": { - "command": "/home/ibrahim/.bun/bin/bunx tsc --build", - "exitCode": 0, - "timedOut": false, - "durationMs": 2175, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853323128, - "lastActivityAtMs": 1787853323128 - }, - "apps/web:lint": { - "command": "/home/ibrahim/.bun/bin/bun run lint", - "exitCode": 0, - "timedOut": false, - "durationMs": 68, - "stdoutTail": "Checked 14 files in 13ms. No fixes applied.\n", - "stderrTail": "$ biome check .\n", - "startedAtMs": 1787853325303, - "lastActivityAtMs": 1787853325366 - }, - "apps/web:format": { - "command": "format (no formatter configured)", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "skipped (tool not configured)", - "stderrTail": "" - }, - "apps/web:test": { - "command": "test (no test script)", - "exitCode": null, - "timedOut": false, - "status": "na", - "durationMs": 0, - "stdoutTail": "n/a", - "stderrTail": "" - }, - "dotnetRestore": { - "command": "dotnet restore IncidentOpsPortal.sln", - "exitCode": 0, - "timedOut": false, - "durationMs": 4856, - "stdoutTail": " Determining projects to restore...\n Restored /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api/IncidentOps.Api.csproj (in 3.6 sec).\n Restored /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/IncidentOps.Api.Tests.csproj (in 3.6 sec).\n", - "stderrTail": "", - "startedAtMs": 1787853325381, - "lastActivityAtMs": 1787853330202 - }, - "dotnetBuild": { - "command": "dotnet build IncidentOpsPortal.sln --no-restore", - "exitCode": 0, - "timedOut": false, - "durationMs": 4238, - "stdoutTail": " IncidentOps.Api -> /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api/bin/Debug/net8.0/IncidentOps.Api.dll\n/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/IncidentApiIntegrationTests.cs(96,33): warning CS8604: Possible null reference argument for parameter 'client' in 'Task HttpClientJsonExtensions.PostAsJsonAsync(HttpClient client, string? requestUri, AcknowledgeIncidentRequest value, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default(CancellationToken))'. [/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/IncidentOps.Api.Tests.csproj]\n IncidentOps.Api.Tests -> /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/bin/Debug/net8.0/IncidentOps.Api.Tests.dll\n\nBuild succeeded.\n\n/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/IncidentApiIntegrationTests.cs(96,33): warning CS8604: Possible null reference argument for parameter 'client' in 'Task HttpClientJsonExtensions.PostAsJsonAsync(HttpClient client, string? requestUri, AcknowledgeIncidentRequest value, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default(CancellationToken))'. [/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/IncidentOps.Api.Tests.csproj]\n 1 Warning(s)\n 0 Error(s)\n\nTime Elapsed 00:00:04.00\n", - "stderrTail": "", - "startedAtMs": 1787853330237, - "lastActivityAtMs": 1787853334451 - }, - "test": { - "command": "dotnet test IncidentOpsPortal.sln --no-build", - "exitCode": 1, - "timedOut": false, - "durationMs": 1171, - "stdoutTail": "Test run for /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/bin/Debug/net8.0/IncidentOps.Api.Tests.dll (.NETCoreApp,Version=v8.0)\nA total of 1 test files matched the specified pattern.\n\n", - "stderrTail": "Testhost process for source(s) '/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/bin/Debug/net8.0/IncidentOps.Api.Tests.dll' exited with error: You must install or update .NET to run this application.\nApp: /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/bin/Debug/net8.0/testhost.dll\nArchitecture: x64\nFramework: 'Microsoft.NETCore.App', version '8.0.0' (x64)\n.NET location: /home/ibrahim/.dotnet/\nThe following frameworks were found:\n 10.0.11 at [/home/ibrahim/.dotnet/shared/Microsoft.NETCore.App]\nLearn more:\nhttps://aka.ms/dotnet/app-launch-failed\nTo install missing framework, download:\nhttps://aka.ms/dotnet-core-applaunch?framework=Microsoft.NETCore.App&framework_version=8.0.0&arch=x64&rid=linux-x64&os=zorin.18\n. Please check the diagnostic logs for more information.\nTest Run Aborted.\n", - "startedAtMs": 1787853334475, - "lastActivityAtMs": 1787853335596 - } - }, - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 6468, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ @biomejs/biome@1.9.4 (v2.5.10 available)\n+ turbo@2.10.12\n\n133 packages installed [6.46s]\n", - "stderrTail": "moved pnpm-workspace.yaml to workspaces in package.json\n[1.85s] migrated lockfile from pnpm-lock.yaml\nSaved lockfile\n", - "startedAtMs": 1787853284002, - "lastActivityAtMs": 1787853290461 - }, - "build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 0, - "timedOut": false, - "durationMs": 18953, - "stdoutTail": "\n • Packages in scope: web\n • Running build in 1 packages\n • Remote caching disabled\n\nweb:build: cache miss, executing 11eac617d2802b8b\nweb:build: \nweb:build: > web@0.1.0 build /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/web\nweb:build: > next build\nweb:build: \nweb:build: ▲ Next.js 14.2.35\nweb:build: \nweb:build: Creating an optimized production build ...\nweb:build: ✓ Compiled successfully\nweb:build: Linting and checking validity of types ...\nweb:build: Collecting page data ...\nweb:build: Generating static pages (0/4) ...\nweb:build: Generating static pages (1/4) \r\nweb:build: Generating static pages (2/4) \r\nweb:build: Generating static pages (3/4) \r\nweb:build: ✓ Generating static pages (4/4)\nweb:build: Finalizing page optimization ...\nweb:build: Collecting build traces ...\nweb:build: \nweb:build: Route (app) Size First Load JS\nweb:build: ┌ ○ / 27.1 kB 114 kB\nweb:build: └ ○ /_not-found 871 B 88.1 kB\nweb:build: + First Load JS shared by all 87.3 kB\nweb:build: ├ chunks/183-271d519d1e0b7743.js 31.7 kB\nweb:build: ├ chunks/3cc391f0-ea67247d5ede1772.js 53.6 kB\nweb:build: └ other shared chunks (total) 1.92 kB\nweb:build: \nweb:build: \nweb:build: ○ (Static) prerendered as static content\nweb:build: \n\n Tasks: 1 successful, 1 total\nCached: 0 cached, 1 total\n Time: 18.857s \n\n", - "stderrTail": "$ turbo run build\n• turbo 2.10.12\n", - "startedAtMs": 1787853290470, - "lastActivityAtMs": 1787853309402 - }, - "checkTypes": { - "command": "/home/ibrahim/.bun/bin/bunx tsc --build", - "exitCode": 0, - "timedOut": false, - "durationMs": 2175, - "stdoutTail": "", - "stderrTail": "", - "startedAtMs": 1787853323128, - "lastActivityAtMs": 1787853323128 - }, - "lint": { - "command": "/home/ibrahim/.bun/bin/bun run lint", - "exitCode": 1, - "timedOut": false, - "durationMs": 236, - "stdoutTail": "Checked 21 files in 59ms. No fixes applied.\nFound 3 errors.\n", - "stderrTail": "/page.ts\":·{\n 70 │ + ······\"size\":·3653,\n 71 │ + ······\"mtime_nanos\":·1787853293691384141,\n 72 │ + ······\"mode\":·436,\n 73 │ + ······\"is_dir\":·false\n 74 │ + ····},\n 75 │ + ····\"apps/web/.next/server/app/_not-found\":·{\n 76 │ + ······\"size\":·0,\n 77 │ + ······\"mtime_nanos\":·0,\n 78 │ + ······\"mode\":·0,\n 79 │ + ······\"is_dir\":·true\n 80 │ + ····},\n 81 │ + ····\"apps/web/.next/server/chunks/font-manifest.json\":·{\n 82 │ + ······\"size\":·2,\n 83 │ + ······\"mtime_nanos\":·1787853293689384118,\n 84 │ + ······\"mode\":·436,\n 85 │ + ······\"is_dir\":·false\n 86 │ + ····},\n 87 │ + ····\"apps/web/.next/static/chunks/pages\":·{\n 88 │ + ······\"size\":·0,\n 89 │ + ······\"mtime_nanos\":·0,\n 90 │ + ······\"mode\":·0,\n 91 │ + ······\"is_dir\":·true\n 92 │ + ····},\n 93 │ + ····\"apps/web/.next/server/server-reference-manifest.json\":·{\n 94 │ + ······\"size\":·84,\n 95 │ + ······\"mtime_nanos\":·1787853293691384141,\n 96 │ + ······\"mode\":·436,\n 97 │ + ······\"is_dir\":·false\n 98 │ + ····},\n 99 │ + ····\"apps/web/.next/server/app/_not-found.meta\":·{\n 100 │ + ······\"size\":·142,\n 101 │ + ······\"mtime_nanos\":·1787853303888165031,\n 102 │ + ······\"mode\":·436,\n 103 │ + ······\"is_dir\":·false\n 104 │ + ····},\n 105 │ + ····\"apps/web/.next/server/app/page.js.nft.json\":·{\n 106 │ + ······\"size\":·1910,\n 107 │ + ······\"mtime_nanos\":·1787853309317556488,\n 108 │ + ······\"mode\":·436,\n 109 │ + ······\"is_dir\":·false\n 110 │ + ····},\n 111 │ + ····\"apps/web/.next/server/app/_not-found/page.js.nft.json\":·{\n 112 │ + ······\"size\":·1907,\n 113 │ + ······\"mtime_nanos\":·1787853309317556488,\n 114 │ + ······\"mode\":·436,\n 115 │ + ······\"is_dir\":·false\n 116 │ + ····},\n 117 │ + ····\"apps/web/.next/images-manifest.json\":·{\n 118 │ + ······\"size\":·511,\n 119 │ + ······\"mtime_nanos\":·1787853304118499280,\n 120 │ + ······\"mode\":·436,\n 121 │ + ······\"is_dir\":·false\n 122 │ + ····},\n 123 │ + ····\"apps/web/.next/server/app/page_client-reference-manifest.js\":·{\n 124 │ + ······\"size\":·6451,\n 125 │ + ······\"mtime_nanos\":·1787853299034443208,\n 126 │ + ······\"mode\":·436,\n 127 │ + ······\"is_dir\":·false\n 128 │ + ····},\n 129 │ + ····\"apps/web/.next/server/middleware-react-loadable-manifest.js\":·{\n 130 │ + ······\"size\":·36,\n 131 │ + ······\"mtime_nanos\":·1787853299032443186,\n 132 │ + ······\"mode\":·436,\n 133 │ + ······\"is_dir\":·false\n 134 │ + ····},\n 135 │ + ····\"apps/web/.next/static/chunks/main-app-2ad5ecf306059106.js\":·{\n 136 │ + ······\"size\":·460,\n 137 │ + ······\"mtime_nanos\":·1787853299033443197,\n 138 │ + ······\"mode\":·436,\n 139 │ + ······\"is_dir\":·false\n 140 │ + ····},\n 141 │ + ····\"apps/web/.next/server/functions-config-manifest.json\":·{\n 142 │ + ······\"size\":·28,\n 143 │ + ······\"mtime_nanos\":·1787853303542492934,\n 144 │ + ······\"mode\":·436,\n 145 │ + ······\"is_dir\":·false\n 146 │ + ····},\n 147 │ + ····\"apps/web/.next/static/chunks\":·{\n 148 │ + ······\"size\":·0,\n 149 │ + ······\"mtime_nanos\":·0,\n 453 more lines truncated\n \n\n./.turbo/cache/11eac617d2802b8b-meta.json format ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n × Formatter would have printed the following content:\n \n 1 │ - {\"hash\":\"11eac617d2802b8b\",\"duration\":18843,\"sha\":\"38ccd8d1d2dcedf7bb5738716af96f7d3142e618\",\"dirty_hash\":null}\n 1 │ + {\n 2 │ + ··\"hash\":·\"11eac617d2802b8b\",\n 3 │ + ··\"duration\":·18843,\n 4 │ + ··\"sha\":·\"38ccd8d1d2dcedf7bb5738716af96f7d3142e618\",\n 5 │ + ··\"dirty_hash\":·null\n 6 │ + }\n 7 │ + \n \n\ncheck ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n\n × Some errors were emitted while running checks.\n \n\nerror: script \"lint\" exited with code 1\n", - "startedAtMs": 1787853309423, - "lastActivityAtMs": 1787853309658 - }, - "format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "test": { - "command": "dotnet test IncidentOpsPortal.sln --no-build", - "exitCode": 1, - "timedOut": false, - "durationMs": 1171, - "stdoutTail": "Test run for /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/bin/Debug/net8.0/IncidentOps.Api.Tests.dll (.NETCoreApp,Version=v8.0)\nA total of 1 test files matched the specified pattern.\n\n", - "stderrTail": "Testhost process for source(s) '/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/bin/Debug/net8.0/IncidentOps.Api.Tests.dll' exited with error: You must install or update .NET to run this application.\nApp: /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/multi-dotnet-ops-gemini-3.7-flash-low-prompt-r01/sb21-multi-dotnet-ops-prompt-low.validate-tmp/apps/api.Tests/bin/Debug/net8.0/testhost.dll\nArchitecture: x64\nFramework: 'Microsoft.NETCore.App', version '8.0.0' (x64)\n.NET location: /home/ibrahim/.dotnet/\nThe following frameworks were found:\n 10.0.11 at [/home/ibrahim/.dotnet/shared/Microsoft.NETCore.App]\nLearn more:\nhttps://aka.ms/dotnet/app-launch-failed\nTo install missing framework, download:\nhttps://aka.ms/dotnet-core-applaunch?framework=Microsoft.NETCore.App&framework_version=8.0.0&arch=x64&rid=linux-x64&os=zorin.18\n. Please check the diagnostic logs for more information.\nTest Run Aborted.\n", - "startedAtMs": 1787853334475, - "lastActivityAtMs": 1787853335596 - }, - "sourceHash": "015d706b6d0d9a992f9c53b4a8e2531da2aeb6ee855299bdb8b22d3b856585ea", - "cacheKey": "af2ea8db7d79e04ebfafe29cfbcc38dc7e01f6a40ff33a6163b59907806839d6", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 17, - "total": 18, - "percent": 94, - "misses": [ - "frontend:shadcn" - ] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "format-failed", - "lint-failed", - "stack-mismatch", - "test-failed" - ], - "outcome": "success" - }, - { - "id": "ts-svelte-edge-orpc-gemini-3.7-flash-low-prompt-r01", - "specId": "ts-svelte-edge-orpc", - "specTitle": "SvelteKit edge app on Cloudflare Workers with Hono + oRPC and D1", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/ts-svelte-edge-orpc-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-ts-svelte-edge-orpc-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/ts-svelte-edge-orpc-gemini-3.7-flash-low-prompt-r01/sb21-ts-svelte-edge-orpc-prompt-low", - "codeMetrics": { - "files": 184, - "lines": 220342, - "bytes": 23065207 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 526396, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 10280, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ typescript@5.9.3 (v7.0.2 available)\n\n331 packages installed [10.27s]\n", - "stderrTail": "moved pnpm-workspace.yaml to workspaces in package.json\n[7.56s] migrated lockfile from pnpm-lock.yaml\nSaved lockfile\n", - "startedAtMs": 1787853273354, - "lastActivityAtMs": 1787853283626 - }, - "build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 1, - "timedOut": false, - "durationMs": 64, - "stdoutTail": "", - "stderrTail": "$ pnpm -r run build\nUnsupported package manager specification (bun@1.3.12)\nerror: script \"build\" exited with code 1\n", - "startedAtMs": 1787853283634, - "lastActivityAtMs": 1787853283697 - }, - "not-run:lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:bun:apps/web": { - "command": "bun validation of apps/web not run: an earlier core step already failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:bun:apps/server": { - "command": "bun validation of apps/server not run: an earlier core step already failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:bun:packages/db": { - "command": "bun validation of packages/db not run: an earlier core step already failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:bun:packages/api": { - "command": "bun validation of packages/api not run: an earlier core step already failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "not-run:bun:packages/schema": { - "command": "bun validation of packages/schema not run: an earlier core step already failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - } - }, - "install": { - "command": "/home/ibrahim/.bun/bin/bun install --concurrent-scripts=2 --network-concurrency=8", - "exitCode": 0, - "timedOut": false, - "durationMs": 10280, - "stdoutTail": "bun install v1.4.0 (34cbb9a40)\n\n+ typescript@5.9.3 (v7.0.2 available)\n\n331 packages installed [10.27s]\n", - "stderrTail": "moved pnpm-workspace.yaml to workspaces in package.json\n[7.56s] migrated lockfile from pnpm-lock.yaml\nSaved lockfile\n", - "startedAtMs": 1787853273354, - "lastActivityAtMs": 1787853283626 - }, - "build": { - "command": "/home/ibrahim/.bun/bin/bun run build", - "exitCode": 1, - "timedOut": false, - "durationMs": 64, - "stdoutTail": "", - "stderrTail": "$ pnpm -r run build\nUnsupported package manager specification (bun@1.3.12)\nerror: script \"build\" exited with code 1\n", - "startedAtMs": 1787853283634, - "lastActivityAtMs": 1787853283697 - }, - "lint": { - "command": "lint not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "format": { - "command": "format not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "test": { - "command": "test not run: an earlier validation step failed", - "exitCode": null, - "timedOut": false, - "status": "skip", - "durationMs": 0, - "stdoutTail": "not run (verdict already determined)", - "stderrTail": "" - }, - "sourceHash": "d92d36fade333166835c34db890131f264d7c676b6d28c2aae0a32cd5b68527a", - "cacheKey": "f67c981788829de8a8f6795b9640ec351a28e2ca981c8129c0c6d6a7ad4867c4", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 13, - "total": 13, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [ - "build-failed", - "format-failed", - "lint-failed", - "test-failed", - "validation-failed" - ], - "outcome": "model-failure" - }, - { - "id": "dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01", - "specId": "dotnet-blazor-cqrs", - "specTitle": ".NET Blazor app with Dapper, Duende IdentityServer, and HotChocolate GraphQL", - "model": "gemini-3.7-flash", - "effort": "low", - "effectiveReasoning": "low", - "path": "prompt", - "trial": 1, - "promptStyle": "explicit", - "runDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01", - "projectName": "sb21-dotnet-blazor-cqrs-prompt-low", - "projectDir": "/home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low", - "codeMetrics": { - "files": 47, - "lines": 1916, - "bytes": 61470 - }, - "claude": { - "exitCode": 0, - "timedOut": false, - "durationMs": 291181, - "spawnError": false, - "stderrTail": "" - }, - "budgetPolicy": { - "budgetEnforced": false, - "maxBudgetUsd": 12 - }, - "provenance": { - "suiteVersion": "3.0", - "harnessVersion": "3.1.0", - "validationCacheVersion": 9, - "promptVersion": "2026-08-21-scaffbench-3.1", - "resourceProfileId": "low-2w-v1", - "agentAdapter": "agy", - "configuredTrials": 1, - "specOrderSeed": 847190081 - }, - "validation": { - "projectExists": true, - "qualityGateRequested": true, - "steps": { - "dotnetRestore": { - "command": "dotnet restore Sb21DotnetBlazorCqrs.slnx", - "exitCode": 0, - "timedOut": false, - "durationMs": 1705, - "stdoutTail": " Determining projects to restore...\n Restored /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/src/Sb21DotnetBlazorCqrs.Web/Sb21DotnetBlazorCqrs.Web.csproj (in 356 ms).\n Restored /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/tests/Sb21DotnetBlazorCqrs.Tests/Sb21DotnetBlazorCqrs.Tests.csproj (in 374 ms).\n", - "stderrTail": "", - "startedAtMs": 1787853335796, - "lastActivityAtMs": 1787853337466 - }, - "dotnetBuild": { - "command": "dotnet build Sb21DotnetBlazorCqrs.slnx --no-restore", - "exitCode": 0, - "timedOut": false, - "durationMs": 5787, - "stdoutTail": " Sb21DotnetBlazorCqrs.Web -> /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/src/Sb21DotnetBlazorCqrs.Web/bin/Debug/net10.0/Sb21DotnetBlazorCqrs.Web.dll\n Sb21DotnetBlazorCqrs.Tests -> /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/tests/Sb21DotnetBlazorCqrs.Tests/bin/Debug/net10.0/Sb21DotnetBlazorCqrs.Tests.dll\n\nBuild succeeded.\n 0 Warning(s)\n 0 Error(s)\n\nTime Elapsed 00:00:05.55\n", - "stderrTail": "", - "startedAtMs": 1787853337501, - "lastActivityAtMs": 1787853343265 - }, - "test": { - "command": "dotnet test Sb21DotnetBlazorCqrs.slnx --no-build", - "exitCode": 0, - "timedOut": false, - "durationMs": 2043, - "stdoutTail": "Test run for /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/tests/Sb21DotnetBlazorCqrs.Tests/bin/Debug/net10.0/Sb21DotnetBlazorCqrs.Tests.dll (.NETCoreApp,Version=v10.0)\nA total of 1 test files matched the specified pattern.\n\nPassed! - Failed: 0, Passed: 7, Skipped: 0, Total: 7, Duration: 304 ms - Sb21DotnetBlazorCqrs.Tests.dll (net10.0)\n", - "stderrTail": "", - "startedAtMs": 1787853343288, - "lastActivityAtMs": 1787853345280 - } - }, - "install": { - "command": "dotnet restore Sb21DotnetBlazorCqrs.slnx", - "exitCode": 0, - "timedOut": false, - "durationMs": 1705, - "stdoutTail": " Determining projects to restore...\n Restored /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/src/Sb21DotnetBlazorCqrs.Web/Sb21DotnetBlazorCqrs.Web.csproj (in 356 ms).\n Restored /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/tests/Sb21DotnetBlazorCqrs.Tests/Sb21DotnetBlazorCqrs.Tests.csproj (in 374 ms).\n", - "stderrTail": "", - "startedAtMs": 1787853335796, - "lastActivityAtMs": 1787853337466 - }, - "build": { - "command": "dotnet build Sb21DotnetBlazorCqrs.slnx --no-restore", - "exitCode": 0, - "timedOut": false, - "durationMs": 5787, - "stdoutTail": " Sb21DotnetBlazorCqrs.Web -> /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/src/Sb21DotnetBlazorCqrs.Web/bin/Debug/net10.0/Sb21DotnetBlazorCqrs.Web.dll\n Sb21DotnetBlazorCqrs.Tests -> /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/tests/Sb21DotnetBlazorCqrs.Tests/bin/Debug/net10.0/Sb21DotnetBlazorCqrs.Tests.dll\n\nBuild succeeded.\n 0 Warning(s)\n 0 Error(s)\n\nTime Elapsed 00:00:05.55\n", - "stderrTail": "", - "startedAtMs": 1787853337501, - "lastActivityAtMs": 1787853343265 - }, - "test": { - "command": "dotnet test Sb21DotnetBlazorCqrs.slnx --no-build", - "exitCode": 0, - "timedOut": false, - "durationMs": 2043, - "stdoutTail": "Test run for /home/ibrahim/code/Better-Fullstack/testing/llm-benchmarks/v3/lane-3/gemini-3-7-flash-low-2026-08-27/runs/dotnet-blazor-cqrs-gemini-3.7-flash-low-prompt-r01/sb21-dotnet-blazor-cqrs-prompt-low.validate-tmp/tests/Sb21DotnetBlazorCqrs.Tests/bin/Debug/net10.0/Sb21DotnetBlazorCqrs.Tests.dll (.NETCoreApp,Version=v10.0)\nA total of 1 test files matched the specified pattern.\n\nPassed! - Failed: 0, Passed: 7, Skipped: 0, Total: 7, Duration: 304 ms - Sb21DotnetBlazorCqrs.Tests.dll (net10.0)\n", - "stderrTail": "", - "startedAtMs": 1787853343288, - "lastActivityAtMs": 1787853345280 - }, - "sourceHash": "5a39c2a1768ef9b120196154f32e8ced0931563b54d3a58f51194db483902733", - "cacheKey": "c5d7c5bf1022ce859f0250a667306a6ad3d6bd3696020f7a40e5f5df0ee8fe8e", - "cacheHit": false, - "deferred": false - }, - "stackScore": { - "matched": 24, - "total": 24, - "percent": 100, - "misses": [] - }, - "toolCompliance": { - "score": 2, - "total": 2, - "checks": [ - { - "id": "no-bf-config", - "status": "pass", - "detail": "prompt-only must not produce bts.jsonc" - }, - { - "id": "no-bf-tool", - "status": "pass", - "detail": "prompt-only must not call a Better-Fullstack MCP tool or CLI" - } - ] - }, - "failureTags": [], - "outcome": "success" - } - ] -} diff --git a/benchmarks/gemini-3-7-flash-low/summary.md b/benchmarks/gemini-3-7-flash-low/summary.md deleted file mode 100644 index e77e3a96c..000000000 --- a/benchmarks/gemini-3-7-flash-low/summary.md +++ /dev/null @@ -1,65 +0,0 @@ -# ScaffBench 3.0 Run - -Harness: 3.1.0 -Agent: Antigravity (single agent; single model family per row) -Specs: ai-search-workbench, rust-leptos-axum, python-ingestion-api, go-realtime-api, multi-dotnet-ops, ts-svelte-edge-orpc, dotnet-blazor-cqrs, multi-ts-go-grpc, java-spring-jooq-keycloak, elixir-broadway-absinthe, react-native-expo, frontier-polyglot-proto, frontier-effect-eventsourcing -Repeats: 1 -Prompt style: explicit - -## Path × effort summary - -This is an ablation across creation paths and reasoning effort for one agent -(Antigravity), not a cross-vendor leaderboard. Pass rate is over *scored* runs. -Provider, harness, and validation infrastructure outcomes are inconclusive and -excluded; budget and generation-deadline exhaustion remain scored failures. - -"Pass@1" is the CORE pass rate, install + build + typecheck + native compile, -i.e. does the project actually build and run. "Quality" is the stricter advisory -tier (core + lint/format; tests, doctor and route run and are reported but -affect no score): a project can be Pass@1-green but Quality-red because it is -mis-formatted or a style-lint warns. Formatting is a quality metric, never a -brokenness verdict, so it does not move Pass@1. "Wired -libs" is scored from the generated artifact (deps + imports + files); -"Faithful" is the assisted-path bts.jsonc-vs-requested diagnostic. - -Reliability is reported per spec, not pooled: "Macro" is the mean of per-spec -pass rates; "pass@k" counts specs solved on at least one repeat and "pass^k" -specs solved on every repeat. The Wilson "CI95" is shown only when a cell has -≥ 8 scored runs (below that it reads `n<8`, since e.g. 3/3 and 0/3 -intervals overlap and the interval is not informative). - -"Index" is the single rankable 0-100 composite the table is sorted by. Each -spec earns a graded score: 60% for a Core pass, 20% for the share of lint and -format gates green (tests are not scored), and 20% for the stack score (wired libs, -traps, restraint). The index is the difficulty-weighted mean of those per-spec -scores (spec difficulty 1, 2, or 3 is pinned in the spec file), times 100. -Latency is median / p95 (wall-clock -moves with provider load, so the mean alone is misleading over small samples). - -| Model | Effort | Effective reasoning | Path | Index | Pass@1 | Quality | Inconclusive | Macro | pass@k | pass^k | CI95 | Wired libs | Faithful | Acceptance | Command discipline | Median / p95 | Avg output tokens | Avg cost | Publication | Failure tags | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | -gemini-3.7-flash | low | low | prompt | 63 | 8/13 | 3/13 | 0 | 62% | 8/13 | 8/13 | 62% (36-82) | 98% | – | – | 100% | 233.2s / 558.9s | | | ranked | test-failed:11, build-failed:5, format-failed:10, install-failed:3, lint-failed:9, typecheck-failed:3, validation-failed:5, stack-mismatch:3 - -## Introduction cohorts - -| Introduced | Specs | Pass@1 | Pass rate | -| --- | ---: | ---: | ---: | -| 2026-08-21 | 13 | 8/13 | 62% | - -## Runs - -| Spec | Trial | Effort | Effective reasoning | Model | Path | Validation | Failure tags | Claude exit | Time | Output tokens | Cost | Wired % | Wired | Faithful | Acceptance | Install | Build | Typecheck | Lint | Test | Validation cache | -| --- | ---: | --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | -elixir-broadway-absinthe | 1 | low | low | gemini-3.7-flash | prompt | pass | test-failed | 0 | 558.9s | | | 100 | 20/20 | – | – | 0 | 0 | | | 1 | miss -python-ingestion-api | 1 | low | low | gemini-3.7-flash | prompt | model-failure | build-failed, format-failed, install-failed, lint-failed, test-failed, typecheck-failed, validation-failed | 0 | 76.1s | | | 100 | 16/16 | – | – | 1 | | | | | miss -frontier-effect-eventsourcing | 1 | low | low | gemini-3.7-flash | prompt | pass | format-failed, lint-failed, test-failed | 0 | 140.0s | | | 100 | 14/14 | – | – | 0 | 0 | 0 | | | miss -react-native-expo | 1 | low | low | gemini-3.7-flash | prompt | model-failure | build-failed, format-failed, lint-failed, test-failed, typecheck-failed, validation-failed | 0 | 258.8s | | | 100 | 13/13 | – | – | 0 | 1 | | | | miss -go-realtime-api | 1 | low | low | gemini-3.7-flash | prompt | pass | format-failed, test-failed | 0 | 136.0s | | | 100 | 16/16 | – | – | 0 | 0 | | 0 | | miss -java-spring-jooq-keycloak | 1 | low | low | gemini-3.7-flash | prompt | pass | | 0 | 207.1s | | | 100 | 28/28 | – | – | | 0 | | | 0 | miss -rust-leptos-axum | 1 | low | low | gemini-3.7-flash | prompt | pass | format-failed, lint-failed, test-failed | 0 | 452.9s | | | 100 | 24/24 | – | – | | 0 | | | | miss -multi-ts-go-grpc | 1 | low | low | gemini-3.7-flash | prompt | model-failure | build-failed, format-failed, install-failed, lint-failed, stack-mismatch, test-failed, validation-failed | 0 | 233.4s | | | 91 | 20/22 | – | – | 1 | | | | | miss -ai-search-workbench | 1 | low | low | gemini-3.7-flash | prompt | model-failure | build-failed, format-failed, install-failed, lint-failed, stack-mismatch, test-failed, typecheck-failed, validation-failed | 0 | 90.7s | | | 85 | 23/27 | – | – | 1 | | | | | miss -frontier-polyglot-proto | 1 | low | low | gemini-3.7-flash | prompt | pass | format-failed, lint-failed, test-failed | 0 | 233.2s | | | 100 | 10/10 | – | – | 0 | 0 | 0 | | | miss -multi-dotnet-ops | 1 | low | low | gemini-3.7-flash | prompt | pass | format-failed, lint-failed, stack-mismatch, test-failed | 0 | 222.7s | | | 94 | 17/18 | – | – | 0 | 0 | 0 | 1 | 1 | miss -ts-svelte-edge-orpc | 1 | low | low | gemini-3.7-flash | prompt | model-failure | build-failed, format-failed, lint-failed, test-failed, validation-failed | 0 | 526.4s | | | 100 | 13/13 | – | – | 0 | 1 | | | | miss -dotnet-blazor-cqrs | 1 | low | low | gemini-3.7-flash | prompt | pass | | 0 | 291.2s | | | 100 | 24/24 | – | – | 0 | 0 | | | 0 | miss diff --git a/docs/README.md b/docs/README.md index f08c72e30..150f70ce8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,7 +28,6 @@ behavior. Reference and completed documents must never override them. - `docs/update-support-policy.md` - rolling update-window contract and current qualification state - `docs/verified-combinations.md` - generated compatibility evidence - `testing/README.md` - production-package and smoke-test workspace -- `benchmarks/README.md` - committed benchmark summaries - `apps/web/content/docs/` - user-facing product documentation ## Maintenance diff --git a/docs/guidelines/README.md b/docs/guidelines/README.md index be0358fad..5be2b13d8 100644 --- a/docs/guidelines/README.md +++ b/docs/guidelines/README.md @@ -27,7 +27,6 @@ Files: - `template-output-and-validation.md` - template conditional logic, generated output validation, sync test discipline, and framework-specific constraints - `remotion-video-style.md` - default visual style, color system, motion rules, and branding for Remotion videos in this project - `design-reading-guide.md` - ordered index of all in-repo design specs (agent skills + BF Remotion style), precedence rules, and verification commands for agents reading “all designs” -- `scaffbench-benchmark.md` - ScaffBench protocol, machine-load constraints, validation semantics, and publication workflow - `adding-new-tool-options/` - complete guide for adding new tool/library options to any ecosystem - `README.md` - master guide: checklists, both scenarios (existing category vs new category), area-by-area file reference, ecosystem rules, naming conventions, common mistakes - `worked-example.md` - complete end-to-end walkthrough adding "opensearch" to Search, with every file diff and new-category wiring patterns diff --git a/docs/guidelines/capability-evidence-levels.md b/docs/guidelines/capability-evidence-levels.md index 3a1d76ea9..672e79e26 100644 --- a/docs/guidelines/capability-evidence-levels.md +++ b/docs/guidelines/capability-evidence-levels.md @@ -45,4 +45,4 @@ assertions, and declared behavior assertions. - A later-stage result cannot skip an earlier prerequisite. - Dependency presence, generated source strings, theoretical compatibility, and skipped checks are never runtime proof. -- ScaffBench evaluates coding agents. It does not raise a product capability evidence level. +- Fixproof evaluates coding agents. It does not raise a product capability evidence level. diff --git a/docs/guidelines/scaffbench-benchmark.md b/docs/guidelines/scaffbench-benchmark.md deleted file mode 100644 index e3fadf92b..000000000 --- a/docs/guidelines/scaffbench-benchmark.md +++ /dev/null @@ -1,171 +0,0 @@ -# ScaffBench benchmark - -Read this before running ScaffBench, benchmarking a model/CLI, adding a spec, changing the harness, or publishing results. This document is the single home for harness rationale. The code stays comment-free; the "why" lives here. - -ScaffBench measures whether an LLM coding agent can scaffold a working, correctly wired fullstack project from a spec. The agent builds the project; the harness installs, builds, type-checks, and native-compiles it, then scores it. - -## Current protocol (ScaffBench 3, the baseline) - -ScaffBench 3 is a blank-page reset. Nothing from the 2.x boards carries over; the 3.0 specs were rewritten on 2026-08-21 (modern toolchain requirements, conflict traps, scenario briefs), so 2.x and 3.0 scores are not comparable. A published row must match ALL of: - -| Field | Value | -| ---------------- | ---------------------------------------- | -| Harness | `3.1.0` | -| Suite | `3.0` (13 core specs, 2026-08-21 cohort) | -| Prompt version | `2026-08-21-scaffbench-3.1` | -| Validator cache | v9 | -| Resource profile | `low-2w-v1` | -| Quality gates | ON (the board metric is the Index) | -| Path | `prompt` only | -| Repeats | 1 (pass@1); `--top-up N` may extend a row | - -The published board data is `apps/web/src/components/scaffbench/scaffbench-3-data.ts`, regenerated by `scripts/benchmarks/build-scaffbench-3-data.ts`. The web board (`apps/web/src/components/home/llm-benchmark-section.tsx`) currently renders `scaffbench-3-board-data.ts`, whose rows and cells are generated from run summaries by script until the publisher is wired to it. The 2.2 board, blog, and build scripts are archived history; do not extend them. - -What changed from 2.2 and why: - -- **Repeats default to 1.** A 13x3 cohort costs three times the tokens and days of validation wall-clock on one laptop. The board is pass@1 and says so. Honesty note that survives the change: the 2026-07-28 noise analysis found single-trial Full scores carry plus/minus 3-4 specs of noise (about 40% of specs flipped between trials). Coarse placement is what a single-trial row can claim. When you need to rank two models within about 3 specs of each other, top up the close rows instead of rerunning the board: `--top-up 3 --out-dir ` adds trials 2 and 3 for every spec the run recorded, in the same out-dir, under the same seed. The launch protocol stays `repeats: 1`; each top-up is appended to `metadata.runProtocol.topUps` with its trial target, spec list, and timestamp, and the added results carry `configuredTrials: N`. `--repeats N` in a fresh out-dir remains the all-rows alternative. Rules that keep a top-up honest: the same `N` for every close row you compare; all specs, not the ones that failed (rerunning failures only and keeping passes at 1/1 inflates the pass rate for every spec, since a passed spec is never given the chance to fail); decide which rows to top up before looking at the second trial. `--specs` restricts a top-up to a subset, but a row whose specs carry different trial counts publishes as `topUp: "partial"` and is exploratory, never ranked. -- **The `cli` creation path is gone.** It measured flag-mapping against a tool nobody scaffolds with that way.`mcp` remains as a hidden opt-in (`--paths mcp`) for occasional tooling studies; it is not part of the methodology and never publishes. -- **Agents may run servers.** The prompt allows starting servers to self-verify, and requires killing every started process before finishing. An agent that wedges itself eats its own idle timeout and scores as a model failure. Validation-side watcher hangs are still a harness concern (see validator notes). -- **The canonical recorder stamps `path: "prompt"`.**`scripts/benchmarks/record-scaffbench-canonical.ts` scaffolds via the local CLI to prove a spec is solvable by the generator. It is not an agent run; the model id `canonical-cli` marks it, and the narrowed path union has no dedicated value for it. -- **The canonical recorder runs `go mod tidy` before validating.** Generated Go apps ship a direct-requires-only `go.mod` and no `go.sum`; BFS's documented setup step tidies them, so the recorder replays that before validation. The shared validator stays strict (`go mod download` + build, tidy advisory-only): an agent-built project must ship a complete module graph itself. - -## Layout - --`scripts/benchmarks/scaffbench-v2.ts` is the entry point: `bun scripts/benchmarks/scaffbench-v2.ts ` (the filename predates 3.0; the harness inside is versioned by `HARNESS_VERSION`). -`scripts/scaffbench/` is the harness: `cli.ts` (flags), `runner.ts` (scheduling/generation/validation loop), `scoring.ts`, `summary.ts`, `validation/` (per-ecosystem validators plus cache), `specs/`, `agents/` (per-CLI adapters plus `routing.ts`). - -- Tests: `scripts/benchmarks/scaffbench-v2-lib.test.ts`, `scripts/scaffbench-hardening*.test.ts`, `scripts/benchmarks/scaffbench-executor.test.ts` (process-group kill, output limit, env scrub, clone/cache). Run all of them after any harness edit. -`scripts/benchmarks/build-scaffbench-3-data.ts` regenerates the 3.0 board wholesale from every dir in its `RUN_SOURCES`. - -## Supported agents - -The driving CLI is inferred from the model-id prefix by `providerForModel()` (`scripts/scaffbench/agents/routing.ts`): - -| Prefix / pattern | Provider | Notes | -| -------------------------------------------------------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `pi/*` | Pi | No MCP support, so `--paths mcp` is a schedule-time error. Pi reports its own USD cost and the harness keeps it: the Claude pricing table is applied only to runs on the Claude harness, which is the only one that emits full input/cache usage. | -| `kilocode/*`, `kilo/*` | Kilo Code | `kilocode/` drives Kilo with an arbitrary provider id (disambiguates from opencode's `openai/*` and Kilo's credit-gated catalog). Adapter strips the prefix. | -| `opencode/*`, `opencode-go/*`, `cloudflare-ai-gateway/*`, `openai/*` | opencode | Free tier, paid Go subscription, gateway passthrough.`--auto` is baked into the adapter (opencode 1.18 renamed it from `--dangerously-skip-permissions`); without it headless runs auto-reject every tool call and scaffold nothing.`--pure` keeps globally installed plugins out of a trial. Kilo ships the same CLI and the same two flags. | -| `*gemini*` | Antigravity (`agy`) | Plain-text output: no token/cost/session data (fields show `–`). Effort is a distinct catalog entry, not a flag: `agy models` lists Low/Medium/High per model (Gemini 3.1 Pro has no Medium). The adapter refuses any other effort, including `default`, rather than quietly rerunning High under a different label. No MCP support, so `--paths mcp` is a schedule-time error. | -| `gpt*`, `o*`, `codex*` | Codex | Effort maps to `model_reasoning_effort`; runs with `--ignore-user-config`.`reasoning_output_tokens` is a breakdown OF `output_tokens` (verified against a live `codex exec --json` run), so only `output_tokens` is priced and displayed. | -| anything else | Claude Code | Default. | - -Adding a CLI = prefix branch in `providerForModel()` plus a `runX` adapter plus `parseXResult` plus a routing test. - -## Running a benchmark, the rules - -**1. Serial only. Never run two benchmarks, or a benchmark plus other heavy work, on this machine at once.** The 2.2 GPT cohort ran three models in parallel and trial 1 scored systematically worst across all three (Full 11/39 vs 19 and 18 in later trials). Load contamination is real and indistinguishable from model quality after the fact. The harness is already serial (generation and validation both run at concurrency 1); keep the machine serial too. This includes other agent sessions mutating the repo mid-run. - -**2. Repeats stay at 1 unless you are settling a close ranking.** The default is one trial per spec, published as pass@1. Reach for `--repeats 3` only when two rows sit within about 3 specs of each other and the order matters, and start a fresh out-dir for it. - -**3. Detach long runs.** Background shells get killed 20-30 min after last output. Pattern: - -```bash -nohup bun scripts/benchmarks/scaffbench-v2.ts --model --efforts high \ - --out-dir testing/llm-benchmarks/v3/-high- \ - > testing/llm-benchmarks/v3/-high-.log 2>&1 & -``` - -**4. Preflight checklist** (missing toolchains score specs inconclusive, silently weakening the row): - --`bun`, `cargo`, `go`, `dotnet`, `mix`, `java`, `uv` all on PATH. - -- The driving CLI installed and authenticated (`opencode auth list`, `codex login status`, ...). -- At least 20 GB free disk; broad sweeps have died on ENOSPC before. -- No other benchmark, validation, or agent session running. -- One run per out-dir, ever (runs are queue-locked per out-dir). - -**5. Post-run checks before believing a summary:** - -- Inconclusive count (`provider-infra`, `harness-infra`, `validation-infra`). These are excluded from the pass denominator; more than a couple means the run needs repair, not publication. -`skip` steps inside quality-tier failures. A skip is a gate that should have run but could not; if it points at a generated/vendored directory, suspect the validator, not the model (that class of bug is what validator v6 fixed). -`deadline-exhausted` /`timeout-stuck` outcomes. Real failures by classification, but eyeball whether the machine (not the model) was wedged. -- Zero-usage results on opencode. The CLI masks 429s as `reason: unknown` with zero tokens; purge and re-run those trials. -`steps = 0` on opencode/Kilo rows is a known artifact (tool-steps are parsed from `claude.stdout.json` only). - -## Flags reference - --`--model ` (default `opus`), `--efforts default|low|medium|high|xhigh|max` (comma list), `--repeats ` (default 1), `--out-dir `, `--max-budget-usd ` (default 12/spec). -`--specs core` (13, default) | comma list |`--list-specs`. -`--paths prompt|mcp`. Default `prompt`;`mcp` is a hidden opt-in outside the methodology. -`--prompt-style explicit|natural`.`natural` is the discovery lane. - -- Two-phase: `--generate-only`, then `--validate-existing` (validation cached by source hash plus cache version).`--force-revalidate` re-runs validation ignoring the cache AND recomputes the marker scores, so a row restamped with the current provenance can never carry a verdict or a wired-libs score from an older harness. Re-running the same out-dir resumes; completed trials are skipped. -- A `--validate-existing` pass reuses the model, efforts, paths, specs, repeats, and prompt style recorded in `summary.json` instead of the current CLI defaults, so re-opening a repeats-3 directory can never relabel it as pass@1 (or restamp it with the default `opus` model). Validation flags (`--force-revalidate`, quality gates, out-dir) still come from the invocation. -- Argument validation is strict. An unknown `--specs`/`--paths`/`--efforts` value, a non-integer `--repeats`, a non-numeric `--max-budget-usd`, and an empty schedule are all hard errors, because each of them otherwise produces a silently partial cohort (or a $0 budget that converts every paid result into `budget-exhausted`). -`--no-quality-gate` opts OUT of quality gates (they default on; a board row without them is unpublishable). The explicit flag always wins: it turns the quality tier off even for a spec whose `validationProfile.qualityGate` is `true`.`--doctor-check --route-check` add root-level advisory checks, and they are eligibility-AND-flag by design: a check runs only when the run asks for it and the spec declares itself a candidate. Neither flag alone is enough, so the specs that cannot answer a doctor or route check are never charged for it. -`--skip-validation`, `--write-matrix-only`, `--repair`. - -Resume rules: `summary.json` is written atomically (temp file plus rename) and an unparsable one is a hard stop, never treated as "no summary", that mistake would forget completed trials and overwrite their archives. A recorded trial counts as done unless its outcome is `provider-infra` or `harness-infra`; a fast terminal failure IS the pass@1 result and is never quietly replaced by a retry. - -Timeouts and budgets (constants.ts): generation 90 min times the spec multiplier (generous so only genuinely stuck agents hit it; a SIGTERM'd thoughtful run loses its cost accounting AND scores as a model failure), 20 min idle, validation 20 min/step, 90 min/project, root cap 12. The $/spec budget is the real cost backstop. - -## Scoring - -- **Index** (0-100): every spec earns a graded score and the index is the difficulty-weighted mean of those scores, times 100. A spec score is 0.6 for a Core pass, 0.2 for the share of lint and format gates green, and 0.2 for the stack score (requested libraries wired, trap and restraint markers respected). Tests are not scored: the harness only runs the tests the model wrote, `cargo test` and `go test` pass with zero test files, and a TS project with no test script records `na`, so the gate rewarded writing nothing. A project that failed Core keeps its stack credit and nothing else, since the quality gates were never run; a run without quality gates, or one whose only lint and format steps belong to a root that never reached Core, earns no quality credit either. Command discipline is reported but weighs nothing on any path. Spec difficulty is pinned in each spec file (`difficulty`: 1 easy, 2 hard, 3 frontier) and only changes with a suite version, so a new row never moves another row's score. Current tiers, set from the 2.2 per-spec Core pass rates (13x3 cells, four models) and the 3.0 traps: 1 for `go-realtime-api`, `python-ingestion-api`, `rust-leptos-axum`, `java-spring-jooq-keycloak`; 2 for `ai-search-workbench`, `ts-svelte-edge-orpc`, `dotnet-blazor-cqrs`, `multi-ts-go-grpc`, `multi-dotnet-ops`, `elixir-broadway-absinthe`, `react-native-expo`; 3 for both frontier specs. The 2.2 and 2.1 boards were computed with the old 75/25 formula and are not restated. -- **Core pass** = install, build, typecheck, and native compile all green. **Quality pass** = Core AND every applicable lint and format gate green; tests run and are reported but affect no score. A `skip` (gate that should have run but no tool was configured) disqualifies;`na` steps are excluded. Quality is a subset of Core always;`passRate === 100` from the raw harness is never trusted directly (it was vacuously 100 on zero-step runs once). -- **Wired libs** = requested libraries actually present (deps, imports, files). The text index that feeds markers is built from EMITTED code and config only;`bts.jsonc` is excluded because it is Better-Fullstack's own manifest, and a stack it merely declares is not a stack the project wired.`bts.jsonc` is still read separately for the faithfulness score, and other config formats (`wrangler.jsonc` and friends) stay indexed. -- **Eligibility**: `MIN_RANKED_TRIALS = 1`, so a single-trial row ranks. Wilson intervals are suppressed below `MIN_CI_RUNS = 8` scored runs so no row over-claims precision. Per-cell `scoredTrials` is authoritative. -- Re-validation under a newer validator refreshes the result's `harnessVersion`/`validationCacheVersion` provenance (the verdict really was produced by the current validator); generation-side provenance (prompt version, adapter, trials, seed) is never touched. - -## The low-footprint validator (3.1.0, profile `low-2w-v1`) - -Validation is serial at the project level but child toolchains default to every logical CPU, which is why concurrency 1 still saturated the machine. Harness 3.1.0 fixes that (full investigation: `testing/scaffbench-validation-footprint-investigation-2026-08-21.md`). The profile is part of the protocol: its id sits in provenance and the validation cache key, and every published row must use the same one. - -- **Two-worker caps via environment** on every validation command: `GOMAXPROCS=2` (Go build `-p` and test `-parallel` default to it), `CARGO_BUILD_JOBS=2`, `UV_CONCURRENT_BUILDS/INSTALLS=2` with `UV_CONCURRENT_DOWNLOADS=8`, `ERL_FLAGS="+S 2:2"`, `JAVA_TOOL_OPTIONS=-XX:ActiveProcessorCount=2` (Maven and Gradle size worker pools from it), `MSBUILDDISABLENODEREUSE=1`. Bun installs run with `--concurrent-scripts=2 --network-concurrency=8`. Node framework builds remain the one uncapped case. -- **macOS background QoS**: every validation command runs under `taskpolicy -c background`, which children inherit and which throttles CPU priority plus disk/network I/O. -- **Raised timeouts to match the caps**: 20 min/step, 90 min/project. Throttling without raising them would manufacture false timeout failures. -- **Process-group termination**: validation commands run detached in their own process group; timeout, output-limit, and completion all SIGTERM the group with a 3 s SIGKILL escalation. This closes the old watcher-hang hole where per-step timeouts killed only the direct child. Generation uses the same `spawnProcessTree` helper, so an agent's tool subprocesses and any server it left running die with it instead of contaminating the next serial trial. -- **Bounded output**: the harness retains a 256 KiB rolling tail per stream and kills the tree if a stream exceeds 16 MiB total; the overflow is recorded as a model-owned step failure. -- **Environment scrubbing**: validation children get a scrubbed environment (credential-shaped variables removed, `GIT_TERMINAL_PROMPT=0`). This is interim exposure reduction, not isolation, generated projects still execute as your user. -- **Fail-fast**: once an executed core step fails, remaining validators are recorded as `not-run:*` skips instead of running (both tiers are already false); within a validator, later steps stop after a core failure and quality gates stop after the first red gate.`unvalidated:*` bookkeeping markers never trigger fail-fast. Planned-but-skipped work is always recorded, never silently omitted. -- **Clone validation**: `validateProjectCached` hashes the pristine archive, then validates a disposable APFS clonefile copy (`cp -Rc`) and deletes it, so lockfiles and build output never mutate the archived evidence or destabilize cache keys. -- **Dependency and build trees never reach the archive**: `deps/` and `_build/` join `node_modules`, `target`, `bin`, `obj` and friends in both the archive filter and the cache-hash skip set. Agents may install and compile before finishing, and an archived `deps/`+`_build/` would let `mix deps.get`/`mix compile` reuse generation-time state instead of proving a clean-source build (and would make the cache key large and nondeterministic). -- **`go mod tidy -diff`** (go >= 1.23) replaces the old copy-then-tidy advisory: non-mutating, exits non-zero when go.mod/go.sum would change. -- **Install alone never passes**: a project whose discovered core surface is only installs (no build/typecheck/compile step anywhere) records `unvalidated:no-build-surface` and fails Core. -- **Toolchain probes** are memoized once per harness run and now cover bun, node, java, mvn, mix, and buf as well. - -Operationally, the lightest pattern is still two-phase: `--generate-only` during the day, then `--validate-existing` on the same out-dir overnight. Changing anything in this profile means a new profile id, a `VALIDATION_CACHE_VERSION` bump, and a fresh canonical + calibration pass; rows validated under different profiles must never share a board. - -## Validator notes - -- Validation is membership-aware: nested `package.json`/`Cargo.toml` roots covered by a parent workspace validate through the parent; uncovered nested roots validate independently. Go modules always validate independently (`go build ./...` never descends into a nested module). -- v6: a `package.json` with none of `name`/`scripts`/`dependencies`/`devDependencies`/`workspaces` is a module-format marker (paraglide's generated output, dist markers), never a root. v5 cost Sol a quality pass this way. -- v9: a prerequisite command whose root (or the project root) holds a `package.json` gets a `bun install` first, recorded as `prerequisite:NN::install`, and then runs with `node_modules/.bin` on `PATH`. Under v8 `buf generate` could never find an npm-installed plugin such as `ts-proto`, so `frontier-polyglot-proto` failed before install for any model. The install is a core step: if it fails, the project fails. -- Java/Elixir validate only on an explicit `validationProfile.native`; file autodetect would run gradle on React Native's `android/` dir. -- An install-only root (no build script, no typecheck surface) measures nothing, so the validator descends into workspace members and the verdict reflects code. -- The format gate resolves a CHECK-mode command from the project's own scripts before it probes for binaries: an explicit `format:check`-style script, a `format` script that already carries a check flag, or `bun run format --check` for a formatter that accepts one (Vite+`vp fmt`, oxfmt); Biome and Prettier binaries remain the fallback. A write-mode formatter always exits 0, which would make the gate vacuous, so the write-flag guard is applied to EVERY candidate script including the check-named ones: a `format:check` script that actually runs `--write` or `--fix` is rejected and the next candidate is tried. Vite+ specs therefore gate through the generated `vp` scripts; if a vp script fails only because of a cwd quirk, that is a template bug, not a spec bug. -- Python validates `pyproject.toml` roots first (`uv sync`), then any remaining `requirements.txt` root (`uv venv` plus `uv pip install -r requirements.txt`, then compileall and an import smoke test from the venv). A pip-style project is a real build surface, not a Core failure. -- A spec's `prerequisiteCommands` entry may carry `whenConfigFound`; the command then runs in each directory holding one of those config files and records an `na` step when none exists.`frontier-polyglot-proto` uses it so a protoc-based or committed-codegen solution is not failed for a missing `buf.gen.yaml`. -`validationProfile` gates: `qualityGate: true` forces the quality tier on for a spec whose run did not ask for it (a programmatic caller such as the canonical recorder), but the explicit `--no-quality-gate` flag still wins: a flag the operator typed is never overridden by spec data.`doctorCheck` and `routeCheckCandidate` mark eligibility, so `--doctor-check` /`--route-check` reach only the specs that declare them. Doctor records `na` when the project has no `bts.jsonc`, it is a Better-Fullstack command, and prompt-only projects have none. -- The validation cache key hashes the EFFECTIVE flags (after the profile merge above), not the flags the run requested, so changing only a spec's `validationProfile` invalidates that spec's entries without a manual cache bump. A result that is not cacheable (timeout, spawn error, repeated transient network failure) deletes any cache file sitting at its key, so a forced revalidation can never leave an older verdict behind for the next ordinary run to resurrect. -- Watcher-shaped builds are killed with their whole process group on step timeout since 3.1.0. If a validation still appears wedged, suspect the harness process itself, not an escaped child. -- .NET specs need the .NET SDK; Antigravity runs never report cost/tokens. - -## Publishing a row - -1. The run must satisfy the protocol table above and have quality gates on. -2. Add the run dir to `RUN_SOURCES` in `build-scaffbench-3-data.ts` (and a `MODEL_LABELS` entry if the slug is ugly). -3. Regenerate wholesale: `bun run scripts/benchmarks/build-scaffbench-3-data.ts`. Every `RUN_SOURCES` dir must exist on disk, and the build refuses loudly unless each source proves its cohort: the exact 13 core spec ids, trials numbered 1..n for every spec (nothing deferred, skipped, or step-less), quality gates recorded ON both in the run options and on every result, and a launch protocol of `repeats: 1`. A row's per-spec cells publish as `{ trials, scored, core, quality, score }`; `qualityPasses` and `corePasses` are macro sums of per-spec pass rates (integers on a pass@1 row, fractions on a topped-up one), `index` sorts the board with `qualityPassPct` as the tie-break, and `trials` plus `topUp` (`none`, `uniform`, `partial`) say how many trials stand behind the row. `qualityGates` and `trialsPerSpec` in the emitted metadata are read back off those runs, never hardcoded. -4. One row per (model, effort). A source dir that ran `low,high` publishes two rows with their own costs and eligibility; efforts are never collapsed into the first one, and two dirs may not claim the same treatment. -5. Commit the run's `summary.md` under `benchmarks/` (summaries only, never scaffolded trees). Archive the raw run dir as a tarball under `testing/llm-benchmarks/archives/`; re-validation after future validator fixes needs the project trees. -6. Tier placement: subscription models that report `cost = $0` (`opencode-go/*`) are PAID. Only genuinely free ids (`*-free`, `:free`) go under the Free divider. - -## Adding or changing a spec - -Calibration rule: run the candidate on a WEAK model (`opencode/deepseek-v4-flash-free`) and a STRONG model; keep it only if weak fails while strong passes. Both pass = saturated, cut it. "Fancy framework" is not "hard spec"; a free model once wired 100% of libs on 11/11 supported specs via MCP. Difficulty comes from traps (right-vs-plausible forks), restraint (penalize over-scaffolding), build-correctness, or pure engineering. - -The 3.0 cohort leans on conflict traps: pgvector bait next to a required Qdrant, NativeWind bait next to required Uniwind, a Node adapter bait on a Workers deploy, Turborepo bait next to a required Vite+ toolchain. When you add a trap, add the matching `forbiddenDeps`/`forbiddenText`/`forbiddenFiles` marker, and check the canonical scaffold does not trip it (`bun scripts/benchmarks/record-scaffbench-canonical.ts --spec `).`forbiddenText` scans file CONTENTS, so a bait identified by its filename (`Dioxus.toml`, `nx.json`) needs `forbiddenFiles`. - -Marker matching rules: `text` requires every pattern (AND) and is case-sensitive;`textAny` requires one of them (OR), which is how naming and casing variants of one concept stay a single marker;`files` matches any path ending in the pattern with `*` covering one segment, so a marker names the evidence (`ent/schema/*.go`) rather than a Better-Fullstack path. The prompt fixes only the outer project directory, so an exact path would penalize valid layouts.`deps` and `forbiddenDeps` match a dependency name exactly, unless the pattern ends in `/`, which matches the whole scope (`@nx/` catches `@nx/devkit` as well as `@nx/workspace`), which is how to forbid a tool that ships as a family of packages. - -A marker is scored on both lanes unless it sets `explicitOnly: true`, which drops it from the natural-lane denominator entirely. Use it when a requirement exists only in the explicit requirements list, `ts-minimal-restraint` asks for Turborepo, but its discovery-lane prompt deliberately never mentions it, so the natural lane must not be charged for omitting a tool it was never asked for. Every other marker, traps included, applies to both lanes. - -The text index that markers scan covers `.html`, `.vue`, and `.svelte` as well as source and config files, so evidence carried by a plain `', - ); - expect((await scoreArtifact(minimalSpec, dir)).misses).toContain("forbidden:analytics"); - - for (const [specId, file, contents, marker] of [ - [ - "elixir-broadway-absinthe", - "mix.exs", - '{:bcrypt_elixir, "~> 3.0"}', - "forbidden:phx-gen-auth", - ], - ["rust-leptos-axum", "Cargo.toml", 'yew = "0.23"', "forbidden:yew"], - [ - "java-spring-jooq-keycloak", - "SecurityConfig.java", - "UserDetailsService userDetailsService()", - "forbidden:in-app-auth", - ], - [ - "dotnet-blazor-cqrs", - "Program.cs", - "builder.Services.AddControllersWithViews();", - "forbidden:mvc", - ], - ] as const) { - const specDir = await tempDirectory("sb-r3-trap-"); - try { - await writeFile(path.join(specDir, file), contents); - const spec = SCAFFBENCH_2_SPECS.find((candidate) => candidate.id === specId)!; - expect((await scoreArtifact(spec, specDir)).misses).toContain(marker); - } finally { - await rm(specDir, { recursive: true, force: true }); - } - } - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); -}); diff --git a/scripts/benchmarks/scaffbench-hardening.test.ts b/scripts/benchmarks/scaffbench-hardening.test.ts deleted file mode 100644 index 8ea4e9121..000000000 --- a/scripts/benchmarks/scaffbench-hardening.test.ts +++ /dev/null @@ -1,780 +0,0 @@ -import * as BunContext from "@effect/platform-bun/BunContext"; -import { - CALIBRATION_WEAK_MODEL, - CLAUDE_TIMEOUT_MS, - GEN_TIMEOUT_MS, - HARNESS_VERSION, - MIN_RANKED_TRIALS, - PROMPT_VERSION, - SCAFFBENCH_2_SPECS, - SCAFFBENCH_SUITE_VERSION, - VALIDATION_CACHE_VERSION, - aggregateResults, - buildGenerationSchedule, - cacheableValidation, - calibrationOptions, - calibrationVerdict, - classifyOutcome, - cohortPassRates, - commandStep, - configuredPythonTypechecker, - dotnetValidationTargets, - expoExportCommand, - findPythonEntryModule, - generationTimeoutMs, - hasTransientNetworkSignature, - hashProjectSource, - SCAFFBENCH_SPEC_SCORE_WEIGHTS, - specDifficulty, - specScore, - parseArgs, - parseClaudeResult, - parseCodexResult, - parseOpencodeResult, - progressEventTime, - providerForModel, - publicationEligibility, - qualityPassed, - repairPromptFor, - rollupOutcome, - runCommand, - seededShuffle, - selectRepairFailure, - specShuffleSeed, - validateCargoProject, - validateGoProject, - validateProject, - validationCacheKey, - writeSummary, - type BenchmarkSpec, - type RunProvenance, - type RunResult, - type ScaffbenchOptions, - type StepResult, -} from "@scaffbench/index"; -import { describe, expect, it } from "bun:test"; -import * as Effect from "effect/Effect"; -import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import path from "node:path"; - -const aiSpec = SCAFFBENCH_2_SPECS.find((spec) => spec.id === "ai-search-workbench")!; -const goSpec = SCAFFBENCH_2_SPECS.find((spec) => spec.id === "go-realtime-api")!; - -const options = (outDir = "/tmp/scaffbench-hardening"): ScaffbenchOptions => ({ - command: "run", - model: "gpt-5.6-sol", - efforts: ["high"], - paths: ["prompt"], - specs: [aiSpec.id], - repeats: 1, - outDir, - maxBudgetUsd: "12", - skipValidation: false, - generateOnly: false, - validateExisting: false, - forceRevalidate: false, - qualityGate: false, - doctorCheck: false, - routeCheck: false, - promptStyle: "explicit", - listSpecs: false, - writeMatrixOnly: false, - repair: false, -}); - -const step = (overrides: Partial = {}): StepResult => ({ - command: "bun run build", - exitCode: 0, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - ...overrides, -}); - -function run(overrides: Partial = {}): RunResult { - return { - id: "r1", - specId: aiSpec.id, - specTitle: aiSpec.title, - model: "gpt-5.6-sol", - effort: "high", - effectiveReasoning: "high", - path: "prompt", - trial: 1, - promptStyle: "explicit", - runDir: "/tmp/run", - projectName: "project", - projectDir: "/tmp/run/project", - claude: { exitCode: 0, timedOut: false, durationMs: 10, outputTokens: 10 }, - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { build: step() }, - }, - stackScore: { matched: 1, total: 1, percent: 100, misses: [] }, - toolCompliance: { score: 2, total: 2, checks: [] }, - failureTags: [], - ...overrides, - }; -} - -const effectPromise = (effect: Effect.Effect): Promise => - Effect.runPromise(effect.pipe(Effect.provide(BunContext.layer)) as Effect.Effect); - -async function tempDirectory(prefix: string) { - return mkdtemp(path.join(tmpdir(), prefix)); -} - -async function executable(filePath: string, body: string) { - await writeFile(filePath, `#!/bin/sh\nset -eu\n${body}\n`); - await chmod(filePath, 0o755); -} - -async function withFakePath(directory: string, action: () => Promise) { - const previous = process.env.PATH; - process.env.PATH = `${directory}${path.delimiter}${previous ?? ""}`; - try { - return await action(); - } finally { - process.env.PATH = previous; - } -} - -const provenance = (overrides: Partial = {}): RunProvenance => ({ - suiteVersion: SCAFFBENCH_SUITE_VERSION, - harnessVersion: HARNESS_VERSION, - validationCacheVersion: VALIDATION_CACHE_VERSION, - promptVersion: PROMPT_VERSION, - agentAdapter: "codex", - configuredTrials: MIN_RANKED_TRIALS, - specOrderSeed: 123, - ...overrides, -}); - -describe("ScaffBench hardening 1: evidence-backed classification", () => { - it("1a derives opencode terminal reasons and recognizes the zero-usage/no-tool signature", () => { - const parsed = parseOpencodeResult( - [ - `{"sessionID":"s1"}`, - `{"part":{"type":"step-finish","reason":"unknown","tokens":{"output":0,"reasoning":0},"cost":0}}`, - ].join("\n"), - ); - expect(parsed?.terminal_reason).toBe("opencode-unknown-zero-usage-no-tools"); - const midFlight = parseOpencodeResult( - [ - `{"sessionID":"s2"}`, - `{"part":{"type":"tool","tool":"bash","state":{"input":{"command":"ls"}}}}`, - `{"part":{"type":"step-finish","reason":"tool-calls","tokens":{"output":588,"reasoning":0},"cost":0}}`, - `{"part":{"type":"step-finish","reason":"unknown","tokens":{"output":0,"reasoning":0},"cost":0}}`, - ].join("\n"), - ); - expect(midFlight?.terminal_reason).toBe("opencode-unknown-zero-usage-step"); - expect( - classifyOutcome( - run({ - projectDir: null, - claude: { - exitCode: 0, - timedOut: false, - durationMs: 1, - outputTokens: 588, - terminalReason: midFlight.terminal_reason, - }, - validation: { projectExists: false, qualityGateRequested: false, steps: {} }, - }), - ), - ).toBe("provider-infra"); - const finished = parseOpencodeResult( - [ - `{"sessionID":"s3"}`, - `{"part":{"type":"step-finish","reason":"tool-calls","tokens":{"output":500,"reasoning":0},"cost":0}}`, - `{"part":{"type":"step-finish","reason":"stop","tokens":{"output":0,"reasoning":0},"cost":0}}`, - ].join("\n"), - ); - expect(finished?.terminal_reason).toBe("stop"); - expect( - classifyOutcome( - run({ - projectDir: null, - claude: { - exitCode: 1, - timedOut: false, - durationMs: 1, - outputTokens: 0, - terminalReason: parsed.terminal_reason, - }, - validation: { projectExists: false, qualityGateRequested: false, steps: {} }, - }), - ), - ).toBe("provider-infra"); - }); - - it("1b persists fine outcomes while preserving the three-way rollup", async () => { - expect(rollupOutcome("provider-infra")).toBe("infra-inconclusive"); - expect(rollupOutcome("budget-exhausted")).toBe("model-failure"); - const dir = await tempDirectory("sb-outcome-"); - try { - const providerFailure = run({ - projectDir: null, - claude: { - exitCode: 1, - timedOut: false, - durationMs: 1, - terminalReason: "provider endpoint unavailable", - }, - validation: { projectExists: false, qualityGateRequested: false, steps: {} }, - }); - await writeSummary(dir, [providerFailure], options(dir), [aiSpec], {}); - const summary = JSON.parse(await readFile(path.join(dir, "summary.json"), "utf8")); - expect(summary.results[0].outcome).toBe("provider-infra"); - expect(summary.aggregates.leaderboard[0].inconclusiveCount).toBe(1); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("1c retries a transient install once, marks recurrence, and keeps registry 404 model-owned", async () => { - const dir = await tempDirectory("sb-retry-"); - try { - const once = path.join(dir, "once"); - const state = path.join(dir, "state"); - await executable(once, `if [ ! -f "$1" ]; then : > "$1"; echo EAI_AGAIN >&2; exit 1; fi`); - const recovered = await effectPromise( - commandStep(once, [state], dir, { retryTransientNetwork: true }), - ); - expect(recovered.exitCode).toBe(0); - expect(recovered.retryCount).toBe(1); - - const always = path.join(dir, "always"); - await executable(always, `echo 'registry.npmjs.org HTTP 503' >&2; exit 1`); - const recurring = await effectPromise( - commandStep(always, [], dir, { retryTransientNetwork: true }), - ); - expect(recurring.retryCount).toBe(1); - expect(recurring.transientNetwork).toBe(true); - expect( - hasTransientNetworkSignature(step({ exitCode: 1, stderrTail: "registry npm HTTP 404" })), - ).toBe(false); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("1d never caches transient failures and always caches green validation", () => { - expect( - cacheableValidation({ - projectExists: true, - qualityGateRequested: false, - steps: { - install: step({ - command: "bun install", - exitCode: 1, - retryCount: 1, - transientNetwork: true, - stderrTail: "EAI_AGAIN", - }), - }, - }), - ).toBe(false); - expect( - cacheableValidation({ - projectExists: true, - qualityGateRequested: false, - steps: { install: step({ command: "bun install" }) }, - }), - ).toBe(true); - }); - - it("1e scores core timeouts as model failures while advisory timeouts preserve a core pass", () => { - const coreTimeout = run({ - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { build: step({ exitCode: null, timedOut: true }) }, - }, - }); - expect(classifyOutcome(coreTimeout)).toBe("model-failure"); - const advisoryTimeout = run({ - validation: { - projectExists: true, - qualityGateRequested: true, - steps: { - build: step(), - test: step({ command: "bun test", exitCode: null, timedOut: true }), - }, - }, - }); - expect(classifyOutcome(advisoryTimeout)).toBe("success"); - expect(qualityPassed(advisoryTimeout)).toBe(true); - }); - - it("1f separates project exit 127 from a spawn-level missing toolchain and records its code", async () => { - const missing = await effectPromise( - runCommand("scaffbench-definitely-missing", [], process.cwd(), 500), - ); - expect(missing.spawnError).toBe(true); - expect(missing.spawnErrorCode).toBeTruthy(); - const project127 = run({ - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { build: step({ exitCode: 127, spawnError: false }) }, - }, - }); - expect(classifyOutcome(project127)).toBe("model-failure"); - }); -}); - -describe("ScaffBench hardening 2: timeout and accounting", () => { - it("2a salvages usage from partial Claude, Codex, and opencode streams", () => { - const claude = parseClaudeResult( - `{"type":"assistant","message":{"usage":{"output_tokens":17}},"session_id":"c1"}`, - ); - expect(claude?.usage.output_tokens).toBe(17); - const codex = parseCodexResult( - [ - `{"type":"thread.started","thread_id":"t"}`, - `{"type":"turn.completed","usage":{"output_tokens":2,"reasoning_output_tokens":3}}`, - `{"type":"turn.completed","usage":{"output_tokens":5,"reasoning_output_tokens":7}}`, - ].join("\n"), - ); - expect(codex?.usage.output_tokens).toBe(5); - const opencode = parseOpencodeResult( - `{"part":{"type":"step-finish","reason":"tool-calls","tokens":{"output":4,"reasoning":6},"cost":1}}`, - ); - expect(opencode?.usage.output_tokens).toBe(10); - }); - - it("2b tags hard timeouts as progressing when a recent tool event was observed", async () => { - expect( - progressEventTime( - `{"type":"item.completed","item":{"type":"command_execution","command":"x"}}`, - 123, - ), - ).toBe(123); - const result = await effectPromise( - runCommand( - "/bin/sh", - [ - "-c", - `printf '%s\\n' '{"type":"item.completed","item":{"type":"command_execution","command":"x"}}'; sleep 1`, - ], - process.cwd(), - 80, - { idleTimeoutMs: 500 }, - ), - ); - expect(result.timeoutKind).toBe("hard"); - expect(result.timeoutProgress).toBe("timeout-progressing"); - }); - - it("2c kills stdout-idle generations separately and tags them stuck", async () => { - const result = await effectPromise( - runCommand("/bin/sh", ["-c", "sleep 1"], process.cwd(), 500, { idleTimeoutMs: 40 }), - ); - expect(result.timedOut).toBe(true); - expect(result.timeoutKind).toBe("idle"); - expect(result.timeoutProgress).toBe("timeout-stuck"); - }); - - it("2d applies per-spec timeout scaling and retains the legacy alias", () => { - expect(generationTimeoutMs({ timeoutMultiplier: 1.5 })).toBe(GEN_TIMEOUT_MS * 1.5); - expect(generationTimeoutMs({})).toBe(GEN_TIMEOUT_MS); - expect(CLAUDE_TIMEOUT_MS).toBe(GEN_TIMEOUT_MS); - }); - - it("2e records enforcement policy and detects post-hoc non-Claude budget exhaustion", () => { - const overBudget = run({ - budgetPolicy: { budgetEnforced: false, maxBudgetUsd: 2 }, - claude: { exitCode: 0, timedOut: false, durationMs: 1, totalCostUsd: 2.51 }, - }); - expect(classifyOutcome(overBudget)).toBe("budget-exhausted"); - expect(overBudget.budgetPolicy).toEqual({ budgetEnforced: false, maxBudgetUsd: 2 }); - }); -}); - -describe("ScaffBench hardening 3: scoring", () => { - it("3a grades a spec as core, lint/format share, and stack; tests carry no weight", () => { - expect(SCAFFBENCH_SPEC_SCORE_WEIGHTS).toEqual({ core: 0.6, quality: 0.2, stack: 0.2 }); - const lintRedTestsRed = run({ - validation: { - projectExists: true, - qualityGateRequested: true, - steps: { - build: step(), - lint: step({ exitCode: 1 }), - format: step(), - test: step({ exitCode: 1 }), - }, - }, - stackScore: { matched: 1, total: 2, percent: 50, misses: ["x"] }, - }); - const graded = specScore(lintRedTestsRed); - expect(graded).toMatchObject({ core: 1, quality: 0.5, stack: 0.5 }); - expect(graded.score).toBeCloseTo(0.8, 10); - - const failedBuild = run({ - validation: { - projectExists: true, - qualityGateRequested: true, - steps: { - build: step({ exitCode: 1 }), - "not-run:lint": step({ exitCode: null, status: "skip" }), - "not-run:format": step({ exitCode: null, status: "skip" }), - }, - }, - }); - expect(specScore(failedBuild)).toMatchObject({ core: 0, quality: 0, stack: 1 }); - }); - - it("3a2 weights the index by pinned spec difficulty, not by spec count", () => { - const easyPass = run({ id: "easy", specId: "go-realtime-api" }); - const frontierFail = run({ - id: "frontier", - specId: "frontier-polyglot-proto", - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { build: step({ exitCode: 1 }) }, - }, - }); - expect(specDifficulty("go-realtime-api")).toBe(1); - expect(specDifficulty("frontier-polyglot-proto")).toBe(3); - expect(aggregateResults([easyPass, frontierFail]).leaderboard[0]?.index).toBe(35); - }); - - it("3b keeps only the stack share for a failed build", () => { - const failed = run({ - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { build: step({ exitCode: 1 }) }, - }, - stackScore: { matched: 1, total: 1, percent: 100, misses: [] }, - toolCompliance: { score: 2, total: 2, checks: [] }, - }); - expect(aggregateResults([failed]).leaderboard[0]?.index).toBe(20); - }); - - it("3c computes wired and discipline components over the same scored trials", () => { - const measured = run({ - id: "measured", - stackScore: { matched: 0, total: 1, percent: 0, misses: ["x"] }, - toolCompliance: { score: 0, total: 2, checks: [] }, - }); - const infra = run({ - id: "infra", - trial: 2, - stackScore: { matched: 1, total: 1, percent: 100, misses: [] }, - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { build: step({ exitCode: 127, spawnError: true, spawnErrorCode: "ENOENT" }) }, - }, - }); - const aggregate = aggregateResults([measured, infra]).leaderboard[0]!; - expect(aggregate.scoredRuns).toBe(1); - expect(aggregate.stackPercent).toBe(0); - expect(aggregate.commandDisciplinePercent).toBe(0); - }); - - it("3d represents an unrequested quality gate as na/null, never pass", () => { - const unrequested = run(); - expect(qualityPassed(unrequested)).toBe("na"); - }); -}); - -describe("ScaffBench hardening 4: trial integrity", () => { - it("4b interleaves repeat rounds and uses a deterministic seeded shuffle", () => { - const specs = [aiSpec, goSpec]; - const schedule = buildGenerationSchedule( - specs, - { repeats: 2, efforts: ["high"], paths: ["prompt"] }, - 99, - ); - expect(schedule.slice(0, 2).every((entry) => entry.trial === 1)).toBe(true); - expect(schedule.slice(2).every((entry) => entry.trial === 2)).toBe(true); - expect(seededShuffle(specs, 99).map((spec) => spec.id)).toEqual( - seededShuffle(specs, 99).map((spec) => spec.id), - ); - expect(specShuffleSeed(options())).toBe(specShuffleSeed(options())); - }); - - it("4c marks only version-consistent rows with at least MIN_RANKED_TRIALS trials as ranked", () => { - const ranked = Array.from({ length: 3 }, (_, index) => - run({ id: `r${index}`, trial: index + 1, provenance: provenance() }), - ); - expect(publicationEligibility(ranked)).toBe("ranked"); - expect( - publicationEligibility([ - ...ranked.slice(0, 2), - run({ id: "mixed", trial: 3, provenance: provenance({ promptVersion: "old" }) }), - ]), - ).toBe("exploratory"); - }); - - it("4c ranks a single-trial row now that MIN_RANKED_TRIALS is 1", () => { - const single = [run({ id: "solo", trial: 1, provenance: provenance({ configuredTrials: 1 }) })]; - expect(publicationEligibility(single)).toBe("ranked"); - expect( - publicationEligibility([ - ...single, - run({ - id: "solo-mixed", - specId: "other-spec", - trial: 1, - provenance: provenance({ configuredTrials: 1, harnessVersion: "0.0.0" }), - }), - ]), - ).toBe("exploratory"); - }); -}); - -describe("ScaffBench hardening 5: validator v4", () => { - it("5a prefers a solution and otherwise exposes every csproj target", async () => { - const dir = await tempDirectory("sb-dotnet-"); - try { - await mkdir(path.join(dir, "a")); - await mkdir(path.join(dir, "b")); - await writeFile(path.join(dir, "a", "A.csproj"), ""); - await writeFile(path.join(dir, "b", "B.csproj"), ""); - expect(await dotnetValidationTargets(dir)).toMatchObject({ kind: "projects" }); - expect((await dotnetValidationTargets(dir)).targets).toHaveLength(2); - await writeFile(path.join(dir, "All.slnx"), ""); - expect(await dotnetValidationTargets(dir)).toEqual({ - kind: "solution", - targets: [path.join(dir, "All.slnx")], - uncoveredProjects: [path.join(dir, "a", "A.csproj"), path.join(dir, "b", "B.csproj")], - }); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("5b runs configured Python typecheckers or finds a src-layout import target", async () => { - const dir = await tempDirectory("sb-python-"); - try { - await writeFile( - path.join(dir, "pyproject.toml"), - "[tool.pyright]\ntypeCheckingMode='strict'\n", - ); - expect(await configuredPythonTypechecker(dir)).toBe("pyright"); - await writeFile(path.join(dir, "pyproject.toml"), "[project]\nname='demo'\n"); - await mkdir(path.join(dir, "src", "demo"), { recursive: true }); - await writeFile(path.join(dir, "src", "demo", "__init__.py"), "import missing_dependency\n"); - expect(await findPythonEntryModule(dir)).toBe("demo"); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("5c invokes cargo check with workspace and all-target coverage", async () => { - const dir = await tempDirectory("sb-cargo-"); - const bin = path.join(dir, "bin"); - const log = path.join(dir, "commands.log"); - try { - await mkdir(bin); - await writeFile(path.join(dir, "Cargo.toml"), "[workspace]\nmembers=[]\n"); - await executable(path.join(bin, "cargo"), `echo "$*" >> "$SCAFFBENCH_TEST_LOG"`); - process.env.SCAFFBENCH_TEST_LOG = log; - await withFakePath(bin, () => effectPromise(validateCargoProject(dir, options(dir)))); - expect(await readFile(log, "utf8")).toContain("check --workspace --all-targets"); - } finally { - delete process.env.SCAFFBENCH_TEST_LOG; - await rm(dir, { recursive: true, force: true }); - } - }); - - it("5d uses go mod download + build and performs tidy only on an advisory copy", async () => { - const dir = await tempDirectory("sb-go-"); - const bin = path.join(dir, "bin"); - const log = path.join(dir, "commands.log"); - const goMod = "module example.com/demo\n\ngo 1.24\n"; - try { - await mkdir(bin); - await writeFile(path.join(dir, "go.mod"), goMod); - await executable(path.join(bin, "go"), `echo "$PWD :: $*" >> "$SCAFFBENCH_TEST_LOG"`); - process.env.SCAFFBENCH_TEST_LOG = log; - const result = await withFakePath(bin, () => - effectPromise(validateGoProject(dir, options(dir))), - ); - const commands = await readFile(log, "utf8"); - expect(commands).toContain("mod download"); - expect(commands).toContain("build ./..."); - expect(commands).toContain("mod tidy"); - expect(result.tidy?.command).toContain("advisory diff"); - expect(await readFile(path.join(dir, "go.mod"), "utf8")).toBe(goMod); - } finally { - delete process.env.SCAFFBENCH_TEST_LOG; - await rm(dir, { recursive: true, force: true }); - } - }); - - it("5e plans a non-interactive Expo export and preserves exit-126 diagnostics", async () => { - expect( - expoExportCommand({ dependencies: { expo: "^55", "react-native-web": "^0.21" } }), - ).toMatchObject({ command: "npx", args: ["expo", "export", "--platform", "web"] }); - expect(expoExportCommand({ dependencies: { expo: "^55" } })?.args).toEqual(["expo", "export"]); - const denied = await effectPromise( - runCommand("/bin/sh", ["-c", "echo 'expo command denied' >&2; exit 126"], process.cwd(), 500), - ); - expect(denied.exitCode).toBe(126); - expect(denied.stderrTail).toContain("expo command denied"); - }); - - it("5f executes spec-declared prerequisites in order before manifest validation", async () => { - const dir = await tempDirectory("sb-prereq-"); - const log = path.join(dir, "order.log"); - try { - const one = path.join(dir, "one"); - const two = path.join(dir, "two"); - await executable(one, `echo one >> "$1"`); - await executable(two, `echo two >> "$1"`); - const spec: BenchmarkSpec = { - ...aiSpec, - id: "prerequisite-test", - prerequisiteCommands: [{ command: [one, log] }, { command: [two, log] }], - }; - const validation = await effectPromise(validateProject(spec, dir, options(dir))); - expect(await readFile(log, "utf8")).toBe("one\ntwo\n"); - expect(Object.keys(validation.steps).slice(0, 2)).toEqual([ - `prerequisite:01:${one}`, - `prerequisite:02:${two}`, - ]); - const frontier = SCAFFBENCH_2_SPECS.find( - (candidate) => candidate.id === "frontier-polyglot-proto", - )!; - expect(frontier.prerequisiteCommands?.[0]?.command).toEqual(["buf", "generate"]); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("5g validates more than three independent roots without a silent cap", async () => { - const dir = await tempDirectory("sb-roots-"); - const bin = path.join(dir, "bin"); - try { - await mkdir(bin); - await executable(path.join(bin, "go"), ":"); - for (let index = 0; index < 4; index += 1) { - const root = path.join(dir, `module-${index}`); - await mkdir(root); - await writeFile(path.join(root, "go.mod"), `module example.com/m${index}\n`); - } - const validation = await withFakePath(bin, () => - effectPromise(validateProject(goSpec, dir, options(dir))), - ); - expect(Object.keys(validation.steps).filter((key) => key.endsWith(":build"))).toHaveLength(4); - expect(Object.keys(validation.steps).some((key) => key.includes("unvalidated"))).toBe(false); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("5h includes environment/toolchains, file modes, and symlink targets in cache identity", async () => { - const dir = await tempDirectory("sb-hash-"); - try { - const file = path.join(dir, "script.sh"); - await writeFile(file, "echo ok\n"); - const first = await Effect.runPromise(hashProjectSource(dir)); - await chmod(file, 0o755); - const second = await Effect.runPromise(hashProjectSource(dir)); - expect(second).not.toBe(first); - await symlink("script.sh", path.join(dir, "link")); - const third = await Effect.runPromise(hashProjectSource(dir)); - expect(third).not.toBe(second); - const base = validationCacheKey( - aiSpec, - options(dir), - third, - { go: "go1" }, - { platform: "x", arch: "a" }, - ); - expect( - validationCacheKey( - aiSpec, - options(dir), - third, - { go: "go2" }, - { platform: "x", arch: "a" }, - ), - ).not.toBe(base); - expect( - validationCacheKey( - aiSpec, - options(dir), - third, - { go: "go1" }, - { platform: "y", arch: "a" }, - ), - ).not.toBe(base); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); -}); - -describe("ScaffBench hardening 6: opt-in batch-4 features", () => { - it("6a keeps repair off by default and builds a bounded same-cell repair prompt", () => { - expect(parseArgs([]).repair).toBe(false); - expect(parseArgs(["--repair"]).repair).toBe(true); - const failing = run({ - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { - build: step({ - exitCode: 1, - stderrTail: Array.from({ length: 80 }, (_, index) => `line-${index}`).join("\n"), - }), - }, - }, - }); - const failure = selectRepairFailure(failing)!; - const prompt = repairPromptFor(failure[0], failure[1]); - const diagnosticLines = prompt.split("Failing-step stderr tail:\n")[1]!.split("\n"); - expect(diagnosticLines).toHaveLength(50); - expect(prompt).toContain("line-79"); - expect(prompt).not.toContain("line-0\n"); - }); - - it("6b parses calibrate and applies the weak-fails/strong-passes keep rule", () => { - const parsed = parseArgs(["calibrate", "--spec", aiSpec.id]); - expect(parsed.command).toBe("calibrate"); - expect(parsed.specs).toEqual([aiSpec.id]); - expect(calibrationOptions(parsed).weak.model).toBe(CALIBRATION_WEAK_MODEL); - expect(calibrationVerdict("model-failure", "success")).toBe("keep"); - expect(calibrationVerdict("success", "success")).toBe("cut"); - expect(calibrationVerdict("provider-infra", "success")).toBe("inconclusive"); - }); - - it("6c backfills ISO introduction dates and reports cohort pass rates", () => { - expect(SCAFFBENCH_2_SPECS.every((spec) => /^\d{4}-\d{2}-\d{2}$/.test(spec.introducedAt))).toBe( - true, - ); - const cohorts = cohortPassRates( - [ - run(), - run({ - id: "failed", - specId: goSpec.id, - validation: { - projectExists: true, - qualityGateRequested: false, - steps: { build: step({ exitCode: 1 }) }, - }, - }), - ], - [aiSpec, goSpec], - ); - expect(cohorts[0]).toMatchObject({ specs: 2, scoredRuns: 2, passCount: 1, passRate: 50 }); - }); -}); - -it("routes kilocode/ ids to the kilo binary with the prefix stripped at invocation", () => { - expect(providerForModel("kilocode/openai/gpt-5.6-luna")).toBe("kilo"); - expect(providerForModel("kilo/openai/gpt-5.6-luna")).toBe("kilo"); - expect(providerForModel("openai/gpt-5.6-luna")).toBe("opencode"); -}); diff --git a/scripts/benchmarks/scaffbench-v2-lib.test.ts b/scripts/benchmarks/scaffbench-v2-lib.test.ts deleted file mode 100644 index b85b487e5..000000000 --- a/scripts/benchmarks/scaffbench-v2-lib.test.ts +++ /dev/null @@ -1,1284 +0,0 @@ -import * as BunContext from "@effect/platform-bun/BunContext"; -import { - aggregateResults, - agentLabelForModel, - canonicalCommand, - claudeCostUsd, - classifyOutcome, - codexCostUsd, - deriveFailureTags, - extractToolUses, - findProjectDir, - parseArgs, - parseClaudeResult, - parseCodexResult, - parseOpencodeResult, - parsePiResult, - piCommandArgs, - piThinkingArgs, - promptFor, - providerForModel, - runCommand, - scoreArtifact, - scoreBts, - scoreProject, - scoreToolCompliance, - SCAFFBENCH_2_SPECS, - typecheckGate, - validationPassed, - qualityPassed, - type RunResult, - type StepResult, -} from "@scaffbench/index"; -import { describe, expect, it } from "bun:test"; -import * as Effect from "effect/Effect"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -const aiSpec = SCAFFBENCH_2_SPECS.find((spec) => spec.id === "ai-search-workbench")!; -const dotnetSpec = SCAFFBENCH_2_SPECS.find((spec) => spec.id === "multi-dotnet-ops")!; - -function stackPart( - id: string, - role: string, - ecosystem: string, - toolId: string, - ownerPartId?: string, -) { - return { id, role, ecosystem, toolId, ownerPartId, source: "selected" }; -} - -function makeRun(overrides: Partial = {}): RunResult { - const base: RunResult = { - id: "ai-search-workbench-claude-opus-4-8-high-mcp-r01", - specId: "ai-search-workbench", - specTitle: aiSpec.title, - model: "claude-opus-4-8", - effort: "high", - effectiveReasoning: "high", - path: "mcp", - trial: 1, - promptStyle: "explicit", - runDir: "/tmp/run", - projectName: "sb21-ai-search-workbench-mcp-high-r01", - projectDir: "/tmp/run/project", - claude: { - exitCode: 0, - timedOut: false, - durationMs: 60_000, - outputTokens: 4000, - totalCostUsd: 1.25, - }, - validation: { - projectExists: true, - qualityGateRequested: true, - steps: { - install: { - command: "bun install", - exitCode: 0, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }, - }, - }, - stackScore: { matched: 24, total: 24, percent: 100, misses: [] }, - toolCompliance: { - score: 2, - total: 2, - checks: [ - { id: "used-mcp", status: "pass", detail: "ok" }, - { id: "no-cli-create", status: "pass", detail: "ok" }, - ], - }, - failureTags: [], - }; - - return { ...base, ...overrides }; -} - -describe("ScaffBench 2 harness config", () => { - it("defaults to a Mac-friendly core matrix", () => { - const options = parseArgs([]); - - expect(options.model).toBe("opus"); - expect(options.efforts).toEqual(["default"]); - expect(options.paths).toEqual(["prompt"]); - expect(options.specs).toEqual([ - "ai-search-workbench", - "rust-leptos-axum", - "python-ingestion-api", - "go-realtime-api", - "multi-dotnet-ops", - "ts-svelte-edge-orpc", - "dotnet-blazor-cqrs", - "multi-ts-go-grpc", - "java-spring-jooq-keycloak", - "elixir-broadway-absinthe", - "react-native-expo", - "frontier-polyglot-proto", - "frontier-effect-eventsourcing", - ]); - expect(options.repeats).toBe(1); - expect(options.promptStyle).toBe("explicit"); - }); - - it("parses generation and validation phase flags", () => { - expect(parseArgs(["--generate-only"]).generateOnly).toBe(true); - expect(parseArgs(["--validate-existing"]).validateExisting).toBe(true); - }); - - it("emits valid non-interactive Better Fullstack command constraints", () => { - const command = canonicalCommand(dotnetSpec, "ops-portal"); - - expect(command).toContain("bun create better-fullstack@latest ops-portal"); - expect(command).toContain("--part frontend:typescript:next"); - expect(command).toContain("--part backend:dotnet:aspnet-minimal"); - expect(command).toContain("--no-install"); - expect(command).toContain("--no-git"); - expect(command).toContain("--disable-analytics"); - }); - - it("keeps creation paths isolated in prompts", () => { - const promptOnly = promptFor(aiSpec, "prompt", "/tmp/run", "workbench", "natural"); - const mcpPrompt = promptFor(aiSpec, "mcp", "/tmp/run", "workbench", "explicit"); - - expect(promptOnly).toContain("Do not use the Better-Fullstack MCP server"); - expect(promptOnly).toContain(aiSpec.naturalPrompt); - expect(mcpPrompt).toContain("bfs_get_guidance"); - expect(mcpPrompt).toContain("bfs_create_project"); - }); - - it("does not leak the canonical command (answer key) into agent-facing prompts", () => { - const mcpPrompt = promptFor(aiSpec, "mcp", "/tmp/run", "workbench", "explicit"); - - expect(mcpPrompt).not.toContain(canonicalCommand(aiSpec, "workbench")); - expect(mcpPrompt).not.toContain("--vector-db"); - expect(mcpPrompt).not.toContain("--shadcn-base"); - }); - - it("records missing spawned tools as failed commands", async () => { - const result = await runCommand( - "scaffbench-missing-binary-for-test", - [], - process.cwd(), - 1_000, - ).pipe(Effect.provide(BunContext.layer), Effect.runPromise); - - expect(result.exitCode).toBe(127); - expect(result.timedOut).toBe(false); - expect(result.spawnError).toBe(true); - expect(result.stderr).toContain("scaffbench-missing-binary-for-test"); - }); -}); - -describe("ScaffBench 2 scoring", () => { - it("scores the existing AI search workbench config exactly", () => { - const score = scoreBts( - aiSpec, - JSON.stringify({ - ...Object.fromEntries(Object.entries(aiSpec.expectedConfig ?? {})), - addons: aiSpec.expectedAddons, - }), - ); - - expect(score).toMatchObject({ matched: 23, total: 23, percent: 100 }); - expect(score.misses).toEqual([]); - }); - - it("scores graph stack parts without relying on flat ecosystem fields", () => { - const backendId = "backend:dotnet:aspnet-minimal"; - const score = scoreBts( - dotnetSpec, - JSON.stringify({ - addons: ["turborepo", "biome", "github-actions"], - stackParts: [ - stackPart("frontend:typescript:next", "frontend", "typescript", "next"), - stackPart( - "frontend:typescript:next.css:typescript:tailwind", - "css", - "typescript", - "tailwind", - "frontend:typescript:next", - ), - stackPart( - "frontend:typescript:next.ui:typescript:shadcn-ui", - "ui", - "typescript", - "shadcn-ui", - "frontend:typescript:next", - ), - stackPart(backendId, "backend", "dotnet", "aspnet-minimal"), - stackPart(`${backendId}.orm:dotnet:ef-core`, "orm", "dotnet", "ef-core", backendId), - stackPart( - `${backendId}.auth:dotnet:aspnet-identity`, - "auth", - "dotnet", - "aspnet-identity", - backendId, - ), - stackPart( - `${backendId}.api:dotnet:minimal-api`, - "api", - "dotnet", - "minimal-api", - backendId, - ), - stackPart(`${backendId}.testing:dotnet:xunit`, "testing", "dotnet", "xunit", backendId), - stackPart( - `${backendId}.testing:dotnet:testcontainers-dotnet`, - "testing", - "dotnet", - "testcontainers-dotnet", - backendId, - ), - stackPart( - `${backendId}.observability:dotnet:serilog`, - "observability", - "dotnet", - "serilog", - backendId, - ), - stackPart( - `${backendId}.realtime:dotnet:signalr`, - "realtime", - "dotnet", - "signalr", - backendId, - ), - stackPart( - `${backendId}.validation:dotnet:fluentvalidation`, - "validation", - "dotnet", - "fluentvalidation", - backendId, - ), - stackPart( - `${backendId}.jobQueue:dotnet:hangfire`, - "jobQueue", - "dotnet", - "hangfire", - backendId, - ), - stackPart( - `${backendId}.caching:dotnet:memory-cache`, - "caching", - "dotnet", - "memory-cache", - backendId, - ), - stackPart(`${backendId}.deploy:dotnet:docker`, "deploy", "dotnet", "docker", backendId), - stackPart("database:universal:postgres", "database", "universal", "postgres"), - ], - }), - ); - - expect(score).toMatchObject({ matched: 19, total: 19, percent: 100 }); - expect(score.misses).toEqual([]); - }); - - it("tags validation and stack failures for summaries", () => { - const failed = makeRun({ - validation: { - projectExists: true, - qualityGateRequested: true, - steps: { - install: { - command: "bun install", - exitCode: 0, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }, - build: { - command: "bun run build", - exitCode: 1, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "build failed", - }, - }, - }, - stackScore: { matched: 22, total: 24, percent: 92, misses: ["vectorDb:qdrant"] }, - }); - - const tags = deriveFailureTags(failed); - - expect(validationPassed(failed)).toBe(false); - expect(tags).toContain("build-failed"); - expect(tags).toContain("validation-failed"); - expect(tags).toContain("stack-mismatch"); - }); - - it("does not pass a run with zero executed validation steps (no manifest fired)", () => { - const empty = makeRun({ validation: { projectExists: true, steps: {} } }); - expect(validationPassed(empty)).toBe(false); - expect(classifyOutcome(empty)).toBe("model-failure"); - }); - - const okStep = (command: string): StepResult => ({ - command, - exitCode: 0, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }); - - it("treats a 'skip' advisory step as a quality miss, not a core failure (Finding 1)", () => { - const run = makeRun({ - validation: { - projectExists: true, - qualityGateRequested: true, - steps: { - install: okStep("bun install"), - build: okStep("bun run build"), - lint: { - command: "lint (no linter configured)", - exitCode: null, - timedOut: false, - status: "skip", - durationMs: 0, - stdoutTail: "skipped (tool not configured)", - stderrTail: "", - }, - }, - }, - }); - expect(validationPassed(run)).toBe(true); - expect(qualityPassed(run)).toBe(false); - }); - - it("scores a format-only failure as a core pass but a quality fail (not broken)", () => { - const run = makeRun({ - validation: { - projectExists: true, - qualityGateRequested: true, - steps: { - install: okStep("bun install"), - build: okStep("bun run build"), - typecheck: okStep("tsc --noEmit"), - format: { - command: "biome format .", - exitCode: 1, - timedOut: false, - durationMs: 1, - stdoutTail: "would reformat 3 files", - stderrTail: "", - }, - }, - }, - }); - expect(validationPassed(run)).toBe(true); - expect(qualityPassed(run)).toBe(false); - expect(classifyOutcome(run)).toBe("success"); - const tags = deriveFailureTags(run); - expect(tags).toContain("format-failed"); - expect(tags).not.toContain("validation-failed"); - }); - - it("excludes an 'na' step (genuinely testless scaffold) from the pass decision", () => { - const run = makeRun({ - validation: { - projectExists: true, - steps: { - install: okStep("bun install"), - build: okStep("bun run build"), - format: okStep("biome format --check ."), - test: { - command: "test (no test script)", - exitCode: null, - timedOut: false, - status: "na", - durationMs: 0, - stdoutTail: "n/a", - stderrTail: "", - }, - }, - }, - }); - expect(validationPassed(run)).toBe(true); - }); - - it("aggregates repeats with pass counts and failure tag counts", () => { - const failed = makeRun({ - id: "ai-search-workbench-claude-opus-4-8-high-mcp-r02", - trial: 2, - validation: { - projectExists: true, - steps: { - install: { - command: "bun install", - exitCode: 1, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "install failed", - }, - }, - }, - failureTags: ["install-failed", "validation-failed"], - }); - - const aggregates = aggregateResults([makeRun(), failed]).leaderboard; - - expect(aggregates).toHaveLength(1); - expect(aggregates[0]).toMatchObject({ - runs: 2, - passCount: 1, - passRate: 50, - failureTags: { "install-failed": 1, "validation-failed": 1 }, - }); - }); -}); - -describe("ScaffBench 2 artifact-grounded scoring", () => { - it("scores libraries wired in the generated artifact, not just declared", async () => { - const dir = await mkdtemp(join(tmpdir(), "sb21-artifact-")); - try { - await writeFile( - join(dir, "package.json"), - JSON.stringify({ dependencies: { hono: "^4", "@qdrant/js-client-rest": "^1" } }), - ); - const spec = { - ...aiSpec, - strictMarkers: [ - { id: "backend:hono", deps: ["hono"] }, - { id: "vectorDb:qdrant", deps: ["@qdrant/js-client-rest"] }, - { id: "search:opensearch", deps: ["@opensearch-project/opensearch"] }, - { id: "forbidden:stripe", forbiddenDeps: ["stripe"] }, - ], - }; - - const score = await scoreArtifact(spec, dir); - - expect(score).toMatchObject({ matched: 3, total: 4 }); - expect(score.misses).toEqual(["search:opensearch"]); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("credits file markers by path suffix and wildcard, not a fixed layout", async () => { - const dir = await mkdtemp(join(tmpdir(), "sb21-files-")); - try { - await mkdir(join(dir, "internal", "ent", "schema"), { recursive: true }); - await writeFile(join(dir, "internal", "ent", "schema", "vehicle.go"), "package schema\n"); - await writeFile( - join(dir, "go.mod"), - "module example.com/fleet\nrequire entgo.io/ent v0.14.0\n", - ); - - const spec = { - ...aiSpec, - strictMarkers: [ - { id: "orm:ent", text: ["entgo.io/ent"], files: ["ent/schema/*.go"] }, - { id: "orm:ent-user-only", text: ["entgo.io/ent"], files: ["ent/schema/user.go"] }, - ], - }; - - const score = await scoreArtifact(spec, dir); - expect(score.misses).toEqual(["orm:ent-user-only"]); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("separates artifact from faithfulness and flags a claimed-but-unwired stack", async () => { - const dir = await mkdtemp(join(tmpdir(), "sb21-unwired-")); - try { - await writeFile( - join(dir, "bts.jsonc"), - JSON.stringify({ - ...Object.fromEntries(Object.entries(aiSpec.expectedConfig ?? {})), - addons: aiSpec.expectedAddons, - }), - ); - await writeFile(join(dir, "package.json"), JSON.stringify({ dependencies: { hono: "^4" } })); - - const { artifact, faithfulness } = await scoreProject(aiSpec, dir); - expect(faithfulness?.percent).toBe(100); - expect(artifact.percent).toBeLessThan(100); - - const run = makeRun({ stackScore: artifact, generatorFaithfulness: faithfulness }); - expect(deriveFailureTags(run)).toContain("stack-unwired"); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); -}); - -describe("ScaffBench 2 run outcomes", () => { - function stepResult( - command: string, - exitCode: number, - opts: { timedOut?: boolean; spawnError?: boolean } = {}, - ) { - return { - command, - exitCode, - timedOut: opts.timedOut ?? false, - spawnError: opts.spawnError ?? false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }; - } - - it("classifies a clean validation pass as success", () => { - expect(classifyOutcome(makeRun())).toBe("success"); - }); - - it("treats deferred validation as inconclusive, not a model failure", () => { - const run = makeRun({ - validation: { projectExists: true, deferred: true, steps: {} }, - }); - - expect(classifyOutcome(run)).toBe("validation-infra"); - expect(deriveFailureTags(run)).toContain("validation-deferred"); - expect(deriveFailureTags(run)).not.toContain("validation-failed"); - }); - - it("classifies an un-spawnable validator binary as infra-inconclusive, not a build break", () => { - const run = makeRun({ - validation: { - projectExists: true, - steps: { cargoCheck: stepResult("cargo check", 127, { spawnError: true }) }, - }, - }); - - expect(classifyOutcome(run)).toBe("harness-infra"); - const tags = deriveFailureTags(run); - expect(tags).toContain("toolchain-missing"); - expect(tags).not.toContain("build-failed"); - }); - - it("treats a child process exiting 127 (broken generated script) as a model-failure", () => { - const run = makeRun({ - validation: { - projectExists: true, - steps: { - install: stepResult("bun install", 0), - build: stepResult("bun run build", 127), - }, - }, - }); - - expect(classifyOutcome(run)).toBe("model-failure"); - const tags = deriveFailureTags(run); - expect(tags).toContain("build-failed"); - expect(tags).not.toContain("toolchain-missing"); - }); - - it("classifies an exhausted token budget as a scored budget failure", () => { - const run = makeRun({ - claude: { - exitCode: 1, - timedOut: false, - durationMs: 60_000, - terminalReason: "max_budget_exhausted", - }, - validation: { projectExists: false, steps: {} }, - }); - - expect(classifyOutcome(run)).toBe("budget-exhausted"); - expect(deriveFailureTags(run)).toContain("budget-exhausted"); - }); - - it("keeps a real build failure as a model-failure, not inconclusive", () => { - const run = makeRun({ - validation: { - projectExists: true, - steps: { install: stepResult("bun install", 0), build: stepResult("bun run build", 1) }, - }, - }); - - expect(classifyOutcome(run)).toBe("model-failure"); - }); - - it("excludes infra-inconclusive runs from the pass-rate denominator", () => { - const ok = makeRun(); - const inconclusive = makeRun({ - id: "ai-search-workbench-claude-opus-4-8-high-mcp-r02", - trial: 2, - validation: { - projectExists: true, - steps: { install: stepResult("uv sync", 127, { spawnError: true }) }, - }, - }); - - const aggregates = aggregateResults([ok, inconclusive]).leaderboard; - - expect(aggregates).toHaveLength(1); - expect(aggregates[0]).toMatchObject({ - runs: 2, - scoredRuns: 1, - inconclusiveCount: 1, - passCount: 1, - passRate: 100, - }); - }); -}); - -describe("ScaffBench 2 statistical reporting", () => { - const passing = () => ({ - projectExists: true, - steps: { - install: { - command: "bun install", - exitCode: 0, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }, - }, - }); - const failing = () => ({ - projectExists: true, - steps: { - build: { - command: "bun run build", - exitCode: 1, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }, - }, - }); - - it("macro-averages per spec and reports pass@k / pass^k instead of one pooled binomial", () => { - const runs = [ - makeRun({ id: "a1", specId: "spec-a", trial: 1, validation: passing() }), - makeRun({ id: "a2", specId: "spec-a", trial: 2, validation: passing() }), - makeRun({ id: "b1", specId: "spec-b", trial: 1, validation: passing() }), - makeRun({ id: "b2", specId: "spec-b", trial: 2, validation: failing() }), - ]; - - const [cell] = aggregateResults(runs).leaderboard; - - expect(cell).toMatchObject({ - scoredRuns: 4, - passCount: 3, - passRate: 75, - macroPassRate: 75, - specCount: 2, - passAnySpecs: 2, - passAllSpecs: 1, - ciReportable: false, - }); - }); - - it("only reports the Wilson CI once a cell reaches the minimum sample size", () => { - const runs = Array.from({ length: 8 }, (_, i) => - makeRun({ id: `r${i}`, specId: `spec-${i % 2}`, trial: i, validation: passing() }), - ); - - const [cell] = aggregateResults(runs).leaderboard; - - expect(cell).toMatchObject({ scoredRuns: 8, ciReportable: true }); - }); - - it("does not count a partially-inconclusive spec as pass^k", () => { - const stepFail127 = { - command: "uv sync", - exitCode: 127, - timedOut: false, - spawnError: true, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }; - const runs = [ - makeRun({ id: "x1", specId: "spec-x", trial: 1, validation: passing() }), - makeRun({ - id: "x2", - specId: "spec-x", - trial: 2, - validation: { projectExists: true, steps: { install: stepFail127 } }, - }), - ]; - - const [cell] = aggregateResults(runs).leaderboard; - - expect(cell).toMatchObject({ - scoredRuns: 1, - inconclusiveCount: 1, - passAnySpecs: 1, - passAllSpecs: 0, - macroPassRate: 100, - }); - }); - - it("reports sub-dollar average cost without integer rounding", () => { - const cheap = (id: string) => - makeRun({ - id, - claude: { exitCode: 0, timedOut: false, durationMs: 1000, totalCostUsd: 0.4 }, - validation: passing(), - }); - - const [cell] = aggregateResults([cheap("c1"), cheap("c2")]).leaderboard; - - expect(cell?.avgCostUsd).toBeCloseTo(0.4, 5); - }); -}); - -describe("ScaffBench 2 resolution robustness", () => { - it("disambiguates the project directory by manifest when a stray dir exists", async () => { - const runDir = await mkdtemp(join(tmpdir(), "sb21-find-")); - try { - await mkdir(join(runDir, "scratch-notes"), { recursive: true }); - await mkdir(join(runDir, "the-project"), { recursive: true }); - await writeFile(join(runDir, "the-project", "package.json"), "{}"); - - expect(await findProjectDir(runDir, "missing-name")).toBe(join(runDir, "the-project")); - } finally { - await rm(runDir, { recursive: true, force: true }); - } - }); - - it("returns the exact project dir when present", async () => { - const runDir = await mkdtemp(join(tmpdir(), "sb21-find-")); - try { - await mkdir(join(runDir, "proj"), { recursive: true }); - expect(await findProjectDir(runDir, "proj")).toBe(join(runDir, "proj")); - } finally { - await rm(runDir, { recursive: true, force: true }); - } - }); - - it("extracts the Claude result JSON despite surrounding noise", () => { - expect(parseClaudeResult('warning: banner\n{"total_cost_usd":1.5}\n')).toMatchObject({ - total_cost_usd: 1.5, - }); - expect(parseClaudeResult('{"a":1}')).toMatchObject({ a: 1 }); - expect(parseClaudeResult("no json here")).toBeNull(); - }); -}); - -describe("ScaffBench 2 composite index", () => { - const failingBuild = () => ({ - projectExists: true, - steps: { - build: { - command: "bun run build", - exitCode: 1, - timedOut: false, - durationMs: 1, - stdoutTail: "", - stderrTail: "", - }, - }, - }); - const stack80 = { matched: 8, total: 10, percent: 80, misses: [] }; - const tool50 = { score: 1, total: 2, checks: [] }; - - it("computes the weighted index and median/p95 latency", () => { - const runs = [ - makeRun({ - id: "i1", - specId: "spec-a", - trial: 1, - claude: { exitCode: 0, timedOut: false, durationMs: 1000 }, - stackScore: stack80, - toolCompliance: tool50, - }), - makeRun({ - id: "i2", - specId: "spec-a", - trial: 2, - claude: { exitCode: 0, timedOut: false, durationMs: 3000 }, - validation: failingBuild(), - stackScore: stack80, - toolCompliance: tool50, - }), - ]; - - const [cell] = aggregateResults(runs).leaderboard; - - expect(cell).toMatchObject({ - macroPassRate: 50, - stackPercent: 80, - commandDisciplinePercent: 50, - index: 46, - medianDurationMs: 1000, - p95DurationMs: 3000, - }); - }); - - it("sorts the leaderboard by index descending", () => { - const high = makeRun({ id: "h", path: "mcp" }); - const low = makeRun({ - id: "l", - path: "prompt", - validation: failingBuild(), - stackScore: { matched: 2, total: 10, percent: 20, misses: [] }, - toolCompliance: { score: 0, total: 2, checks: [] }, - }); - - const board = aggregateResults([high, low]).leaderboard; - - expect(board[0]?.path).toBe("mcp"); - expect(board[0]?.index ?? 0).toBeGreaterThan(board[1]?.index ?? 0); - }); -}); - -describe("ScaffBench 2 discovery lane", () => { - const rustSpec = SCAFFBENCH_2_SPECS.find((spec) => spec.id === "rust-leptos-axum")!; - - it("drops the library names in the natural lane for specs with acceptance sets", () => { - const naturalAi = promptFor(aiSpec, "prompt", "/tmp/run", "wb", "natural"); - const explicitAi = promptFor(aiSpec, "prompt", "/tmp/run", "wb", "explicit"); - const naturalRust = promptFor(rustSpec, "prompt", "/tmp/run", "wb", "natural"); - - expect(naturalAi).not.toContain("Important scoring rule"); - expect(naturalAi).not.toContain("Qdrant"); - expect(explicitAi).toContain("Important scoring rule"); - expect(naturalRust).toContain("Important scoring rule"); - }); - - it("falls back to tsc --noEmit so typecheck cannot be dodged", () => { - expect(typecheckGate({ "check-types": "tsc" }, true)).toBe("check-types"); - expect(typecheckGate({ typecheck: "tsc" }, true)).toBe("typecheck"); - expect(typecheckGate({}, true)).toBe("tsc"); - expect(typecheckGate({}, false)).toBeNull(); - }); - - it("credits an accepted alternative library (pgvector for semantic search)", async () => { - const dir = await mkdtemp(join(tmpdir(), "sb21-accept-")); - try { - await writeFile( - join(dir, "package.json"), - JSON.stringify({ - dependencies: { - next: "*", - hono: "*", - "drizzle-orm": "*", - "better-auth": "*", - ai: "*", - pgvector: "*", - meilisearch: "*", - bullmq: "*", - pino: "*", - vitest: "*", - i18next: "*", - turbo: "*", - }, - }), - ); - await mkdir(join(dir, ".github", "workflows"), { recursive: true }); - await writeFile(join(dir, ".github", "workflows", "ci.yml"), "name: ci"); - - const { acceptance } = await scoreProject(aiSpec, dir, "natural"); - expect(acceptance).toMatchObject({ matched: 13, total: 13 }); - - const explicit = await scoreProject(aiSpec, dir, "explicit"); - expect(explicit.acceptance).toBeUndefined(); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); -}); - -describe("ScaffBench 2 restraint spec", () => { - const minimalSpec = SCAFFBENCH_2_SPECS.find((spec) => spec.id === "ts-minimal-restraint")!; - - it("exists as an opt-in extended spec (not in the default core suite)", () => { - expect(minimalSpec.lane).toBe("extended"); - expect(parseArgs([]).specs).not.toContain("ts-minimal-restraint"); - }); - - it("penalizes an over-engineered project but passes a lean one", async () => { - const overDir = await mkdtemp(join(tmpdir(), "sb21-over-")); - const leanDir = await mkdtemp(join(tmpdir(), "sb21-lean-")); - try { - await writeFile( - join(overDir, "package.json"), - JSON.stringify({ - dependencies: { react: "*", vite: "*", tailwindcss: "*", hono: "*", "better-auth": "*" }, - }), - ); - const over = await scoreArtifact(minimalSpec, overDir); - expect(over.misses).toContain("forbidden:backend"); - expect(over.misses).toContain("forbidden:auth"); - expect(over.misses).not.toContain("frontend:react-vite"); - - await writeFile( - join(leanDir, "package.json"), - JSON.stringify({ - dependencies: { react: "*", vite: "*", tailwindcss: "*" }, - devDependencies: { turbo: "*" }, - }), - ); - const lean = await scoreArtifact(minimalSpec, leanDir); - expect(lean.matched).toBe(lean.total); - } finally { - await rm(overDir, { recursive: true, force: true }); - await rm(leanDir, { recursive: true, force: true }); - } - }); -}); - -describe("ScaffBench 2 acceptance matching precision (Codex #258)", () => { - it("does not credit acceptance from substrings (ai⊂tailwindcss, vite⊂vitest)", async () => { - const dir = await mkdtemp(join(tmpdir(), "sb21-substr-")); - try { - await writeFile( - join(dir, "package.json"), - JSON.stringify({ dependencies: { tailwindcss: "*", vitest: "*" } }), - ); - const { acceptance } = await scoreProject(aiSpec, dir, "natural"); - expect(acceptance?.misses).toContain("ai"); - expect(acceptance?.misses).toContain("web-framework"); - expect(acceptance?.misses).not.toContain("testing"); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); - - it("counts a no-project discovery run as 0 acceptance in the average", () => { - const wired = makeRun({ - id: "acc-a", - acceptanceScore: { matched: 12, total: 12, percent: 100, misses: [] }, - }); - const noProject = makeRun({ - id: "acc-b", - acceptanceScore: { matched: 0, total: 12, percent: 0, misses: ["project not found"] }, - }); - - const [cell] = aggregateResults([wired, noProject]).leaderboard; - - expect(cell?.acceptancePercent).toBe(50); - }); -}); - -describe("ScaffBench 2 command discipline from trajectory (P2)", () => { - const streamJson = (...events: object[]) => - `${events.map((event) => JSON.stringify(event)).join("\n")}\n`; - const claudeOutput = (stdout: string): any => ({ - command: "claude", - exitCode: 0, - timedOut: false, - spawnError: false, - durationMs: 1, - stdout, - stderr: "", - stdoutTail: "", - stderrTail: "", - }); - const bashUse = (command: string) => ({ - type: "assistant", - message: { content: [{ type: "tool_use", name: "Bash", input: { command } }] }, - }); - const mcpUse = (name: string) => ({ - type: "assistant", - message: { content: [{ type: "tool_use", name, input: {} }] }, - }); - const resultEvent = { - type: "result", - subtype: "success", - total_cost_usd: 1.2, - usage: { output_tokens: 3000 }, - session_id: "abc", - }; - - it("parses the result line and tool_use events from a stream-json transcript", () => { - const out = streamJson(bashUse("bun create better-fullstack@2.1.1 app --dry-run"), resultEvent); - expect(parseClaudeResult(out)).toMatchObject({ total_cost_usd: 1.2, session_id: "abc" }); - const uses = extractToolUses(out); - expect(uses).toHaveLength(1); - expect(uses[0]).toMatchObject({ name: "Bash" }); - expect(uses[0]?.command).toContain("--dry-run"); - }); - - it("fails a prompt-only run that shells out to the BF CLI", async () => { - const out = streamJson(bashUse("bun create better-fullstack@2.1.1 app"), resultEvent); - const tc = await scoreToolCompliance("prompt", null, claudeOutput(out)); - expect(tc.checks.find((check) => check.id === "no-bf-tool")?.status).toBe("fail"); - }); - - it("passes the MCP path when bfs_create_project is actually called", async () => { - const out = streamJson(mcpUse("mcp__better-fullstack__bfs_create_project"), resultEvent); - expect(await scoreToolCompliance("mcp", null, claudeOutput(out))).toMatchObject({ - score: 2, - total: 2, - }); - }); -}); - -describe("ScaffBench 2 acceptance scoped-prefix (Codex #261)", () => { - it("matches a scoped-prefix pattern like @auth/ against @auth/core", async () => { - const dir = await mkdtemp(join(tmpdir(), "sb21-authjs-")); - try { - await writeFile( - join(dir, "package.json"), - JSON.stringify({ dependencies: { "@auth/core": "*", "@auth/drizzle-adapter": "*" } }), - ); - const { acceptance } = await scoreProject(aiSpec, dir, "natural"); - expect(acceptance?.misses).not.toContain("auth"); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); -}); - -describe("codex (GPT) agent adapter", () => { - it("routes models to the right provider", () => { - expect(providerForModel("gpt-5.5")).toBe("codex"); - expect(providerForModel("o3")).toBe("codex"); - expect(providerForModel("codex-mini")).toBe("codex"); - expect(providerForModel("claude-opus-4-7")).toBe("claude"); - expect(providerForModel("opus")).toBe("claude"); - }); - - it("parses codex JSONL usage + session, leaving cost undefined", () => { - const jsonl = [ - `{"type":"thread.started","thread_id":"t-123"}`, - `{"type":"item.completed","item":{"type":"agent_message","text":"done"}}`, - `{"type":"turn.completed","usage":{"input_tokens":1000,"cached_input_tokens":800,"output_tokens":120,"reasoning_output_tokens":30}}`, - ].join("\n"); - const parsed = parseCodexResult(jsonl); - expect(parsed?.session_id).toBe("t-123"); - expect(parsed?.usage?.output_tokens).toBe(120); - expect(parsed?.total_cost_usd).toBeUndefined(); - expect(codexCostUsd("gpt-5.6-luna", { output_tokens: 120, reasoning_output_tokens: 30 })).toBe( - (120 * 6) / 1_000_000, - ); - expect(parseCodexResult("not json")).toBeNull(); - }); - - it("extractToolUses understands codex mcp_tool_call + command_execution", () => { - const jsonl = [ - `{"type":"item.completed","item":{"type":"mcp_tool_call","server":"bfs","tool":"bfs_create_project","arguments":{}}}`, - `{"type":"item.completed","item":{"type":"command_execution","command":"/bin/zsh -lc 'bun create better-fullstack app --dry-run'","exit_code":0}}`, - ].join("\n"); - const uses = extractToolUses(jsonl); - expect(uses.some((u) => /bfs_create_project/i.test(u.name))).toBe(true); - const bash = uses.filter((u) => /(^|_)bash$/i.test(u.name)).map((u) => u.command ?? ""); - expect(bash.some((c) => /create\s+better-fullstack/.test(c))).toBe(true); - expect(bash.some((c) => c.includes("--dry-run"))).toBe(true); - }); -}); - -describe("opencode / Kilo Code agent adapter", () => { - it("routes opencode/* and kilo/* models to their providers", () => { - expect(providerForModel("opencode/north-mini-code-free")).toBe("opencode"); - expect(providerForModel("kilo/poolside/laguna-m.1:free")).toBe("kilo"); - expect(providerForModel("gpt-5.5")).toBe("codex"); - expect(providerForModel("claude-opus-4-8")).toBe("claude"); - }); - - it("routes the paid Go subscription and Cloudflare gateway tiers to opencode", () => { - expect(providerForModel("opencode-go/deepseek-v4-pro")).toBe("opencode"); - expect(providerForModel("opencode-go/glm-5.2")).toBe("opencode"); - expect(providerForModel("cloudflare-ai-gateway/anthropic/claude-opus-4-8")).toBe("opencode"); - expect(agentLabelForModel("opencode-go/deepseek-v4-pro")).toBe("opencode"); - }); - - it("labels the driving agent from the model (no more hardcoded 'Claude Code')", () => { - expect(agentLabelForModel("claude-opus-4-8")).toBe("Claude Code"); - expect(agentLabelForModel("gpt-5.5")).toBe("Codex"); - expect(agentLabelForModel("opencode/north-mini-code-free")).toBe("opencode"); - expect(agentLabelForModel("kilo/nvidia/nemotron-3-super-120b-a12b:free")).toBe("Kilo Code"); - }); - - it("parses opencode JSONL session, summed tokens, and cost", () => { - const jsonl = [ - `{"sessionID":"ses_0faabff1","type":"session.updated"}`, - `{"part":{"type":"text","text":"thinking"}}`, - `{"part":{"type":"step-finish","tokens":{"output":40,"reasoning":10},"cost":0}}`, - `{"part":{"type":"step-finish","tokens":{"output":3,"reasoning":0},"cost":0}}`, - ].join("\n"); - const parsed = parseOpencodeResult(jsonl); - expect(parsed?.session_id).toBe("ses_0faabff1"); - expect(parsed?.usage?.output_tokens).toBe(53); - expect(parsed?.total_cost_usd).toBe(0); - expect(parseOpencodeResult("not json")).toBeNull(); - }); - - it("extractToolUses understands opencode tool parts (mcp + bash)", () => { - const jsonl = [ - `{"part":{"type":"tool","tool":"better-fullstack_bfs_create_project","state":{"status":"completed"}}}`, - `{"part":{"type":"tool","tool":"bash","state":{"input":{"command":"bun create better-fullstack app --dry-run"}}}}`, - ].join("\n"); - const uses = extractToolUses(jsonl); - expect(uses.some((u) => /bfs_create_project/i.test(u.name))).toBe(true); - const bash = uses.filter((u) => /(^|_)bash$/i.test(u.name)).map((u) => u.command ?? ""); - expect(bash.some((c) => /create\s+better-fullstack/.test(c))).toBe(true); - expect(bash.some((c) => c.includes("--dry-run"))).toBe(true); - }); -}); - -describe("Pi agent adapter", () => { - it("routes bare OpenAI ids through opencode and pi/* ids through Pi", () => { - expect(providerForModel("openai/gpt-5.6-luna")).toBe("opencode"); - expect(providerForModel("OPENAI/gpt-5.6-luna")).toBe("opencode"); - expect(providerForModel("pi/openai-codex/gpt-5.6-luna")).toBe("pi"); - expect(agentLabelForModel("pi/openai-codex/gpt-5.6-luna")).toBe("Pi"); - expect(providerForModel("gpt-5.5")).toBe("codex"); - expect(providerForModel("claude-opus-4-8")).toBe("claude"); - }); - - it("maps effort to --thinking and keeps default omitted", () => { - expect(piThinkingArgs("default")).toEqual([]); - for (const effort of ["low", "medium", "high", "xhigh", "max"] as const) { - expect(piThinkingArgs(effort)).toEqual(["--thinking", effort]); - } - expect( - piCommandArgs({ - prompt: "build it", - model: "pi/openai-codex/gpt-5.6-luna", - effort: "low", - }), - ).toEqual([ - "-p", - "--mode", - "json", - "--provider", - "openai-codex", - "--model", - "gpt-5.6-luna", - "--thinking", - "low", - "--no-session", - "--no-extensions", - "--no-context-files", - "build it", - ]); - }); - - it("parses a trimmed real Pi 0.80.10 JSON capture without double-counting summaries", () => { - const first = { - role: "assistant", - content: [{ type: "toolCall", id: "call_1", name: "bash", arguments: { command: "pwd" } }], - usage: { - input: 3689, - output: 18, - cacheRead: 0, - cacheWrite: 0, - reasoning: 0, - totalTokens: 3707, - cost: { input: 0.003689, output: 0.000108, cacheRead: 0, cacheWrite: 0, total: 0.003797 }, - }, - stopReason: "toolUse", - }; - const second = { - role: "assistant", - content: [{ type: "text", text: "OK" }], - usage: { - input: 151, - output: 5, - cacheRead: 3584, - cacheWrite: 0, - reasoning: 0, - totalTokens: 3740, - cost: { - input: 0.000151, - output: 0.00003, - cacheRead: 0.0003584, - cacheWrite: 0, - total: 0.0005394, - }, - }, - stopReason: "stop", - }; - const jsonl = [ - { type: "session", version: 3, id: "019f74be-26b1-74bf-86c7-ba67026aa61e" }, - { type: "message_end", message: first }, - { - type: "tool_execution_start", - toolCallId: "call_1", - toolName: "bash", - args: { command: "pwd" }, - }, - { type: "message_end", message: second }, - { type: "agent_end", messages: [first, second], willRetry: false }, - ] - .map((event) => JSON.stringify(event)) - .join("\n"); - - const parsed = parsePiResult(jsonl); - expect(parsed?.session_id).toBe("019f74be-26b1-74bf-86c7-ba67026aa61e"); - expect(parsed?.usage?.output_tokens).toBe(23); - expect(parsed?.total_cost_usd).toBeCloseTo(0.0043364, 10); - expect(parsed?.tool_events).toBe(1); - expect(extractToolUses(jsonl)).toContainEqual({ name: "bash", command: "pwd" }); - expect(parsePiResult("not json")).toBeNull(); - }); - - it("adds reasoning tokens and classifies zero-usage/no-tool runs as provider infra", () => { - const withReasoning = parsePiResult( - [ - JSON.stringify({ type: "session", version: 3, id: "pi-reasoning" }), - JSON.stringify({ - type: "message_end", - message: { - role: "assistant", - content: [{ type: "text", text: "done" }], - usage: { output: 7, reasoning: 3, cost: { total: 0.01 } }, - stopReason: "stop", - }, - }), - ].join("\n"), - ); - expect(withReasoning?.usage?.output_tokens).toBe(10); - - const zeroUsage = parsePiResult( - [ - JSON.stringify({ type: "session", version: 3, id: "pi-zero" }), - JSON.stringify({ - type: "message_end", - message: { - role: "assistant", - content: [], - usage: { output: 0, reasoning: 0, cost: { total: 0 } }, - stopReason: "stop", - }, - }), - ].join("\n"), - ); - expect(zeroUsage?.terminal_reason).toBe("pi-zero-usage-no-tools"); - expect( - classifyOutcome( - makeRun({ - projectDir: null, - claude: { - exitCode: 1, - timedOut: false, - durationMs: 1, - outputTokens: zeroUsage?.usage?.output_tokens, - terminalReason: zeroUsage?.terminal_reason, - }, - validation: { projectExists: false, qualityGateRequested: false, steps: {} }, - }), - ), - ).toBe("provider-infra"); - }); -}); - -describe("ScaffBench 2 Claude cost pricing", () => { - it("prices Claude from token usage so a $0 CLI cost is not treated as free", () => { - const usage = { - input_tokens: 1_000_000, - output_tokens: 1_000_000, - cache_read_input_tokens: 1_000_000, - cache_creation_input_tokens: 1_000_000, - }; - expect(claudeCostUsd("claude-opus-4-8", usage)).toBeCloseTo(36.75, 5); - expect(claudeCostUsd("opus", { output_tokens: 1_000_000 })).toBeCloseTo(25, 5); - expect(claudeCostUsd("claude-sonnet-4-6", { input_tokens: 1_000_000 })).toBeCloseTo(3, 5); - expect(claudeCostUsd("claude-haiku-4-5", { output_tokens: 1_000_000 })).toBeCloseTo(5, 5); - }); - - it("returns undefined for non-Claude models so they keep their own reported cost", () => { - expect(claudeCostUsd("gpt-5.5", { output_tokens: 1_000_000 })).toBeUndefined(); - expect(claudeCostUsd("opencode/north-mini", { output_tokens: 1_000 })).toBeUndefined(); - expect(claudeCostUsd("claude-opus-4-8", undefined)).toBeUndefined(); - }); -}); diff --git a/scripts/benchmarks/scaffbench-v2.ts b/scripts/benchmarks/scaffbench-v2.ts deleted file mode 100644 index 3e1adebde..000000000 --- a/scripts/benchmarks/scaffbench-v2.ts +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bun - -import * as BunContext from "@effect/platform-bun/BunContext"; -import * as BunRuntime from "@effect/platform-bun/BunRuntime"; -import * as Effect from "effect/Effect"; - -import { parseArgs, runScaffbench } from "@scaffbench/index"; - -if (import.meta.main) { - BunRuntime.runMain( - runScaffbench(parseArgs(process.argv.slice(2))).pipe(Effect.provide(BunContext.layer)), - ); -} diff --git a/scripts/evidence/validate-public-evidence-claims.ts b/scripts/evidence/validate-public-evidence-claims.ts index c22cbc09d..2970f1e59 100644 --- a/scripts/evidence/validate-public-evidence-claims.ts +++ b/scripts/evidence/validate-public-evidence-claims.ts @@ -52,8 +52,8 @@ export function validatePublicEvidenceContracts(inputs: PublicEvidenceContractIn ) { errors.push("the public badge must fail closed unless the receipt is verified"); } - if (!inputs.pageSource.includes("ScaffBench") || !inputs.pageSource.includes("does not raise")) { - errors.push("the public verification page must keep ScaffBench separate from product evidence"); + if (!inputs.pageSource.includes("Fixproof") || !inputs.pageSource.includes("does not raise")) { + errors.push("the public verification page must keep Fixproof separate from product evidence"); } return errors; } diff --git a/scripts/scaffbench/EFFECT-NOTES.md b/scripts/scaffbench/EFFECT-NOTES.md deleted file mode 100644 index e85439866..000000000 --- a/scripts/scaffbench/EFFECT-NOTES.md +++ /dev/null @@ -1,39 +0,0 @@ -# ScaffBench Effect migration - -## Converted shell - -- `agents/command.ts` now executes child processes with `@effect/platform/Command`. It preserves the existing `CommandResult` contract: rendered command, `number | null` exit code, timeout and spawn-error flags, full stdout/stderr, 4,000-character tails, and wall-clock duration. Stdin is closed with an empty stream, stdout and stderr are drained concurrently, timeout sends `SIGTERM`, and a still-running process is escalated to `SIGKILL` after three seconds. -- Agent adapters now return Effect programs instead of starting promises. The opencode/Kilo MCP config write uses the platform `FileSystem` service. -- Validation commands now compose the shared command Effect without nested runtimes. Cache hashing, reads, and writes live in `validation/cache.ts`; the cache payload, version input, key fields, ignored directories, file ordering, and symlink behavior are unchanged. -- `runner.ts` is an `Effect.gen` orchestration program. Spec, effort, path, trial, and pending-validation traversal use `Effect.forEach` with `concurrency: 1`, preserving the harness's strictly sequential order. -- Runner filesystem work uses the platform `FileSystem` service. The queue lock is acquired and released with `Effect.acquireRelease` inside a `Scope`, so normal completion, failure, or interruption removes the lock. `QUEUE_POLL_MS` and `STALE_LOCK_MS` behavior remains unchanged. -- `scaffbench-v2.ts` supplies `BunContext.layer` and runs the program with `BunRuntime.runMain` only at the executable edge. Tests that directly exercise an Effect supply the same Bun layer explicitly. -- Scoring, prompts, specs, CLI parsing, and summary data transforms remain ordinary functions. Existing summary JSON/Markdown renderers and their byte formatting were not changed; only metadata command probes pass through the Effect command layer. - -## Effect patterns - -- `Command.make`, `Command.start`, and scoped process cleanup -- `FileSystem.FileSystem` service with `BunContext.layer` -- `Effect.gen` for readable imperative-style composition -- `Effect.forEach(..., { concurrency: 1 })` for deterministic orchestration -- `Effect.acquireRelease` plus `Effect.scoped` for lock cleanup -- `Effect.forkScoped` for concurrent stdout/stderr draining -- Tagged `AgentSpawnError` and `ValidationTimeout` values at the command boundary -- `BunRuntime.runMain` at the application edge - -## Dependencies - -Added as root dev dependencies with `bun add -d`: - -- `@effect/platform` `0.96.2` -- `@effect/platform-bun` `0.90.0` - -The existing root `effect` dependency is `3.21.4`. - -## Verification - -- `bun test scripts/ --timeout 1200000`: 74 passed, 0 failed. -- `bun run scripts/benchmarks/splice-scaffbench-2-1.ts` followed by a diff of `apps/web/src/components/home/scaffbench-2-1-data.ts`: zero diff. -- `bun run scripts/benchmarks/scaffbench-v2.ts --list-specs`: 13 specs. -- `bunx tsc --noEmit -p scripts/tsconfig.json`: 31 pre-existing errors remain versus 40 at baseline; no new errors. All four former `agents/command.ts` child-process typing errors are gone. -- Required Python smoke revalidation: completed with `DONE python-ingestion-api-gpt-5.6-luna-medium-mcp validation=true cache=miss`. diff --git a/scripts/scaffbench/agents/agy.ts b/scripts/scaffbench/agents/agy.ts deleted file mode 100644 index c9b993fba..000000000 --- a/scripts/scaffbench/agents/agy.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { CommandExecutor } from "@effect/platform/CommandExecutor"; -import type { CommandResult, Effort } from "@scaffbench/types"; -import type * as Effect from "effect/Effect"; - -import { agentRunCommandOptions, runCommand } from "@scaffbench/agents/command"; -import { GEN_TIMEOUT_MS } from "@scaffbench/constants"; - -const AGY_MODEL_VARIANTS: Record< - string, - { label: string; tiers: Partial> } -> = { - "gemini-3.7-flash": { - label: "Gemini 3.7 Flash", - tiers: { low: "Low", medium: "Medium", high: "High" }, - }, - "gemini-3.6-flash": { - label: "Gemini 3.6 Flash", - tiers: { low: "Low", medium: "Medium", high: "High" }, - }, - "gemini-3.5-flash": { - label: "Gemini 3.5 Flash", - tiers: { low: "Low", medium: "Medium", high: "High" }, - }, - "gemini-3.1-pro": { - label: "Gemini 3.1 Pro", - tiers: { low: "Low", high: "High" }, - }, -}; - -export function agyModelString(model: string, effort: Effort): string { - const variant = AGY_MODEL_VARIANTS[model]; - if (!variant) { - throw new Error( - `agy model ${JSON.stringify(model)} is not in the verified Antigravity catalog ` + - `(${Object.keys(AGY_MODEL_VARIANTS).join(", ")}); run \`agy models\` and update AGY_MODEL_VARIANTS`, - ); - } - const tier = variant.tiers[effort]; - if (!tier) { - throw new Error( - `agy model ${model} has no distinct ${effort} variant; ` + - `supported efforts: ${Object.keys(variant.tiers).join(", ")}`, - ); - } - return `${variant.label} (${tier})`; -} - -export function runAgy(input: { - cwd: string; - prompt: string; - model: string; - effort: Effort; - timeoutMs?: number; -}): Effect.Effect { - return runCommand( - "agy", - [ - "-p", - input.prompt, - "--model", - agyModelString(input.model, input.effort), - "--dangerously-skip-permissions", - "--add-dir", - input.cwd, - "--print-timeout", - `${Math.ceil((input.timeoutMs ?? GEN_TIMEOUT_MS) / 60_000)}m`, - ], - input.cwd, - input.timeoutMs ?? GEN_TIMEOUT_MS, - agentRunCommandOptions("agy"), - ); -} - -export function parseAgyResult(_stdout: string): undefined { - return undefined; -} diff --git a/scripts/scaffbench/agents/claude.ts b/scripts/scaffbench/agents/claude.ts deleted file mode 100644 index 44148d490..000000000 --- a/scripts/scaffbench/agents/claude.ts +++ /dev/null @@ -1,182 +0,0 @@ -import type { CommandExecutor } from "@effect/platform/CommandExecutor"; -import type { CommandResult, Effort } from "@scaffbench/types"; -import type * as Effect from "effect/Effect"; - -import { agentRunCommandOptions, runCommand } from "@scaffbench/agents/command"; -import { GEN_TIMEOUT_MS } from "@scaffbench/constants"; - -export function runClaude(input: { - cwd: string; - prompt: string; - model: string; - effort: Effort; - maxBudgetUsd: string; - mcpConfig: string; - timeoutMs?: number; -}): Effect.Effect { - const effortArgs = input.effort === "default" ? [] : ["--effort", input.effort]; - - return runCommand( - "claude", - [ - "-p", - "--model", - input.model, - ...effortArgs, - // stream-json (requires --verbose) emits the full tool_use trajectory, so - // command discipline can be scored on actual tool calls rather than greps - // of the final result envelope. The final {"type":"result"} line carries - // the same cost/usage/session fields as --output-format json. - "--output-format", - "stream-json", - "--verbose", - "--permission-mode", - "bypassPermissions", - "--no-session-persistence", - "--strict-mcp-config", - "--mcp-config", - input.mcpConfig, - "--max-budget-usd", - input.maxBudgetUsd, - input.prompt, - ], - input.cwd, - input.timeoutMs ?? GEN_TIMEOUT_MS, - agentRunCommandOptions("claude"), - ); -} -// Published token pricing (USD per 1M tokens) for Claude models. The Claude Code -// CLI reports total_cost_usd = 0 on subscription / Max-plan usage (no per-token -// API billing), which makes Claude look like the cheapest, most "reliable" row -// on the leaderboard purely because it shows as free. So we price Claude from -// token usage ourselves - exactly as the Codex path does - to get an -// API-equivalent cost comparable across vendors. Source: Anthropic API pricing, -// 2026. Cache reads ≈ 0.1× input; cache writes (5-min TTL) ≈ 1.25× input. -const CLAUDE_PRICING: Record = { - "claude-fable-5": { input: 10, output: 50 }, - "claude-opus-5": { input: 5, output: 25 }, - "claude-opus-4-8": { input: 5, output: 25 }, - "claude-opus-4-7": { input: 5, output: 25 }, - "claude-opus-4-6": { input: 5, output: 25 }, - "claude-opus-4-5": { input: 5, output: 25 }, - "claude-sonnet-5": { input: 3, output: 15 }, - "claude-sonnet-4-6": { input: 3, output: 15 }, - "claude-haiku-4-5": { input: 1, output: 5 }, - // Bare aliases the harness/CLI may pass through as the model string. - fable: { input: 10, output: 50 }, - opus: { input: 5, output: 25 }, - sonnet: { input: 3, output: 15 }, - haiku: { input: 1, output: 5 }, -}; - -function claudePricingFor(model: string) { - const key = model.toLowerCase(); - if (CLAUDE_PRICING[key]) return CLAUDE_PRICING[key]; - // Fall back to the longest matching family key (e.g. "claude-opus-4-8[1m]" or - // a dated suffix still prices as opus); prefer the most specific match. - const match = Object.keys(CLAUDE_PRICING) - .filter((k) => key.includes(k)) - .sort((a, b) => b.length - a.length)[0]; - return match ? CLAUDE_PRICING[match] : undefined; -} - -type ClaudeUsage = { - input_tokens?: number; - output_tokens?: number; - cache_creation_input_tokens?: number; - cache_read_input_tokens?: number; -}; - -/** Estimated API-equivalent USD cost from Claude token usage × published - * pricing. Returns undefined when the model isn't a priced Claude model, so - * non-Claude providers (Codex/opencode) keep their own reported cost. Cache - * reads are priced at 0.1× input, cache writes at 1.25× input. */ -export function claudeCostUsd(model: string, usage: ClaudeUsage | undefined): number | undefined { - if (!usage) return undefined; - const price = claudePricingFor(model); - if (!price) return undefined; - const input = usage.input_tokens ?? 0; - const output = usage.output_tokens ?? 0; - const cacheRead = usage.cache_read_input_tokens ?? 0; - const cacheWrite = usage.cache_creation_input_tokens ?? 0; - return ( - (input * price.input + - output * price.output + - cacheRead * price.input * 0.1 + - cacheWrite * price.input * 1.25) / - 1_000_000 - ); -} -export function parseClaudeResult(stdout: string): any | null { - // stream-json: the final {"type":"result",...} line carries cost/usage/session. - const lines = stdout.trim().split("\n"); - let partialUsage: ClaudeUsage | undefined; - let partialMetadata: any | null = null; - for (const line of lines) { - const candidate = line.trim(); - if (!candidate.startsWith("{")) continue; - try { - const event = JSON.parse(candidate); - if (event?.type === "result") return event; - // Assistant stream messages report per-message deltas. Summing every - // usage-bearing message preserves timeout accounting; taking only the - // last message undercounts a multi-message partial trajectory. - const usage = event?.message?.usage ?? (event?.type === "assistant" ? event?.usage : null); - if (usage) { - partialUsage = addClaudeUsage(partialUsage, usage); - partialMetadata = { - type: "partial-result", - total_cost_usd: - event?.total_cost_usd ?? - event?.message?.total_cost_usd ?? - partialMetadata?.total_cost_usd, - session_id: - event?.session_id ?? event?.message?.session_id ?? partialMetadata?.session_id, - duration_ms: event?.duration_ms ?? partialMetadata?.duration_ms, - terminal_reason: - event?.terminal_reason ?? event?.stop_reason ?? partialMetadata?.terminal_reason, - }; - } - } catch {} - } - // A SIGTERM can arrive before Claude emits its final result envelope. Salvage - // the sum of every usage-bearing assistant event instead of dropping or - // undercounting the partial trajectory. - if (partialUsage) return { ...partialMetadata, usage: partialUsage }; - // Fallback for --output-format json (single object) or noisy output. - try { - return JSON.parse(stdout); - } catch { - // Tolerate leading/trailing non-JSON (banner lines, warnings) by extracting - // the outermost {...} span before giving up. - const start = stdout.indexOf("{"); - const end = stdout.lastIndexOf("}"); - if (start >= 0 && end > start) { - try { - return JSON.parse(stdout.slice(start, end + 1)); - } catch {} - } - const line = stdout - .trim() - .split("\n") - .reverse() - .find((candidate) => candidate.trim().startsWith("{") && candidate.trim().endsWith("}")); - if (!line) return null; - try { - return JSON.parse(line.trim()); - } catch { - return null; - } - } -} - -function addClaudeUsage(current: ClaudeUsage | undefined, next: ClaudeUsage): ClaudeUsage { - return { - input_tokens: (current?.input_tokens ?? 0) + (next.input_tokens ?? 0), - output_tokens: (current?.output_tokens ?? 0) + (next.output_tokens ?? 0), - cache_creation_input_tokens: - (current?.cache_creation_input_tokens ?? 0) + (next.cache_creation_input_tokens ?? 0), - cache_read_input_tokens: - (current?.cache_read_input_tokens ?? 0) + (next.cache_read_input_tokens ?? 0), - }; -} diff --git a/scripts/scaffbench/agents/codex.ts b/scripts/scaffbench/agents/codex.ts deleted file mode 100644 index 68a83645e..000000000 --- a/scripts/scaffbench/agents/codex.ts +++ /dev/null @@ -1,149 +0,0 @@ -import type { CommandExecutor } from "@effect/platform/CommandExecutor"; -import type { CommandResult, Effort } from "@scaffbench/types"; -import type * as Effect from "effect/Effect"; - -import { agentRunCommandOptions, runCommand } from "@scaffbench/agents/command"; -import { bfSpec, GEN_TIMEOUT_MS } from "@scaffbench/constants"; - -export function runCodex(input: { - cwd: string; - prompt: string; - model: string; - effort: Effort; - useMcp: boolean; - bunx: string; - timeoutMs?: number; -}): Effect.Effect { - const effortArgs = - input.effort === "default" ? [] : ["-c", `model_reasoning_effort=${input.effort}`]; - const mcpArgs = input.useMcp - ? [ - "-c", - `mcp_servers.bfs.command=${JSON.stringify(input.bunx)}`, - "-c", - `mcp_servers.bfs.args=${JSON.stringify([bfSpec("create-better-fullstack"), "mcp"])}`, - ] - : []; - return runCommand( - "codex", - [ - "exec", - "--json", - "-m", - input.model, - ...effortArgs, - "--dangerously-bypass-approvals-and-sandbox", - "--skip-git-repo-check", - "--ignore-user-config", - "-C", - input.cwd, - ...mcpArgs, - input.prompt, - ], - input.cwd, - input.timeoutMs ?? GEN_TIMEOUT_MS, - agentRunCommandOptions("codex"), - ); -} -type CodexUsage = { - input_tokens?: number; - cached_input_tokens?: number; - output_tokens?: number; - reasoning_output_tokens?: number; -}; - -const CODEX_PRICING: Record = { - "gpt-5.5": { input: 5, cachedInput: 0.5, output: 30 }, - "gpt-5.6-sol": { input: 5, cachedInput: 0.5, output: 30 }, - "gpt-5.6-terra": { input: 2.5, cachedInput: 0.25, output: 15 }, - "gpt-5.6-luna": { input: 1, cachedInput: 0.1, output: 6 }, -}; - -function codexPricingFor(model: string) { - const key = model.toLowerCase(); - return ( - CODEX_PRICING[key] ?? - CODEX_PRICING[Object.keys(CODEX_PRICING).find((k) => key.startsWith(k)) ?? ""] - ); -} - -export function codexCostUsd(model: string, usage: CodexUsage): number | undefined { - const price = codexPricingFor(model); - if (!price) return undefined; - const input = usage.input_tokens ?? 0; - const cached = usage.cached_input_tokens ?? 0; - const output = usage.output_tokens ?? 0; - return ( - (Math.max(0, input - cached) * price.input + - cached * price.cachedInput + - output * price.output) / - 1_000_000 - ); -} - -export function parseCodexResult(stdout: string, model?: string): any | null { - const usageEvents: CodexUsage[] = []; - let threadId: string | undefined; - let sawUsage = false; - let terminalReason: string | undefined; - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (!trimmed.startsWith("{")) continue; - let event: any; - try { - event = JSON.parse(trimmed); - } catch { - continue; - } - if (event?.type === "thread.started" && typeof event.thread_id === "string") { - threadId = event.thread_id; - } - if ( - event?.usage && - (event?.type === "turn.completed" || /(?:^|\.)usage(?:\.|$)/.test(event?.type ?? "")) - ) { - usageEvents.push(event.usage); - sawUsage = true; - } - if (event?.type === "turn.failed" || event?.type === "error") { - terminalReason = event?.error?.message ?? event?.message ?? event?.reason ?? event?.type; - } - } - if (!sawUsage && !threadId && !terminalReason) return null; - const cumulative = usageEvents.every( - (usage, index) => index === 0 || isUsageSuperset(usageEvents[index - 1]!, usage), - ); - const usage = cumulative - ? usageEvents.at(-1) - : usageEvents.reduce(addCodexUsage, undefined); - const outputTokens = usage !== undefined ? (usage.output_tokens ?? 0) : undefined; - return { - type: "result", - usage: outputTokens !== undefined ? { output_tokens: outputTokens } : undefined, - total_cost_usd: - usage !== undefined && model !== undefined ? codexCostUsd(model, usage) : undefined, - session_id: threadId, - duration_ms: undefined, - terminal_reason: terminalReason, - }; -} - -function isUsageSuperset(previous: CodexUsage, next: CodexUsage) { - const fields = [ - "input_tokens", - "cached_input_tokens", - "output_tokens", - "reasoning_output_tokens", - ] as const; - return fields.every((field) => (next[field] ?? 0) >= (previous[field] ?? 0)); -} - -function addCodexUsage(current: CodexUsage | undefined, next: CodexUsage): CodexUsage { - return { - input_tokens: (current?.input_tokens ?? 0) + (next.input_tokens ?? 0), - cached_input_tokens: (current?.cached_input_tokens ?? 0) + (next.cached_input_tokens ?? 0), - output_tokens: (current?.output_tokens ?? 0) + (next.output_tokens ?? 0), - reasoning_output_tokens: - (current?.reasoning_output_tokens ?? 0) + (next.reasoning_output_tokens ?? 0), - }; -} diff --git a/scripts/scaffbench/agents/command.ts b/scripts/scaffbench/agents/command.ts deleted file mode 100644 index 804a436d4..000000000 --- a/scripts/scaffbench/agents/command.ts +++ /dev/null @@ -1,267 +0,0 @@ -import type { CommandResult } from "@scaffbench/types"; - -import { GEN_IDLE_TIMEOUT_MS, TIMEOUT_PROGRESS_WINDOW_MS } from "@scaffbench/constants"; -import { spawnProcessTree } from "@scaffbench/process-tree"; -import * as Effect from "effect/Effect"; - -export type RunCommandOptions = { - idleTimeoutMs?: number; - env?: Record; -}; - -export type IdleCapableAdapter = "claude" | "codex" | "opencode" | "kilo" | "agy" | "pi"; - -export function agentRunCommandOptions( - adapter: IdleCapableAdapter, - idleTimeoutMs = GEN_IDLE_TIMEOUT_MS, -): RunCommandOptions { - return adapter === "agy" ? {} : { idleTimeoutMs }; -} - -export function runCommand( - command: string, - args: readonly string[], - cwd: string, - timeoutMs: number, - options: RunCommandOptions = {}, -): Effect.Effect { - const displayCommand = [command, ...args].map(quoteArg).join(" "); - - return Effect.promise( - () => - new Promise((resolve) => { - const started = Date.now(); - let stdout = ""; - let stderr = ""; - let lastActivityAtMs = started; - let lastStdoutActivityAtMs = started; - let lastStderrActivityAtMs = started; - let lastProgressActivityAtMs: number | undefined; - let stdoutLineRemainder = ""; - let stderrLineRemainder = ""; - let timedOut = false; - let timeoutKind: "hard" | "idle" | undefined; - let settled = false; - const inFlightToolIds = new Set(); - - const appendOutput = (stream: "stdout" | "stderr", chunk: Buffer) => { - const receivedAt = Date.now(); - lastActivityAtMs = receivedAt; - const text = chunk.toString(); - if (stream === "stdout") { - lastStdoutActivityAtMs = receivedAt; - stdout += text; - } else { - lastStderrActivityAtMs = receivedAt; - stderr += text; - } - const remainder = stream === "stdout" ? stdoutLineRemainder : stderrLineRemainder; - const lines = `${remainder}${text}`.split("\n"); - if (stream === "stdout") stdoutLineRemainder = lines.pop() ?? ""; - else stderrLineRemainder = lines.pop() ?? ""; - for (const line of lines) { - const activity = streamEventActivity(line, receivedAt); - if (activity.progressAtMs !== undefined) { - lastProgressActivityAtMs = activity.progressAtMs; - } - for (const id of activity.startedToolIds) inFlightToolIds.add(id); - for (const id of activity.completedToolIds) inFlightToolIds.delete(id); - } - }; - - const settle = (result: CommandResult) => { - if (settled) return; - settled = true; - clearTimeout(hardTimer); - if (idleTimer !== undefined) clearInterval(idleTimer); - resolve(result); - }; - - const tree = spawnProcessTree( - command, - args, - { cwd, env: { ...process.env, ...options.env } }, - { - onStdout: (chunk) => appendOutput("stdout", chunk), - onStderr: (chunk) => appendOutput("stderr", chunk), - onError: (cause) => { - const message = `${displayCommand}: ${formatSpawnError(cause)}`; - settle({ - command: displayCommand, - exitCode: 127, - timedOut: false, - spawnError: true, - spawnErrorCode: spawnErrorCode(cause), - durationMs: Date.now() - started, - stdout: "", - stderr: message, - stdoutTail: "", - stderrTail: tail(message), - startedAtMs: started, - lastActivityAtMs, - }); - }, - onClose: (code) => { - tree.kill(); - const finished = Date.now(); - settle({ - command: displayCommand, - exitCode: timedOut ? null : code, - timedOut, - timeoutKind, - spawnError: false, - durationMs: finished - started, - stdout, - stderr, - stdoutTail: tail(stdout), - stderrTail: tail(stderr), - timeoutProgress: timedOut - ? timeoutKind === "idle" || - lastProgressActivityAtMs === undefined || - (inFlightToolIds.size === 0 && - finished - lastProgressActivityAtMs > TIMEOUT_PROGRESS_WINDOW_MS) - ? "timeout-stuck" - : "timeout-progressing" - : undefined, - startedAtMs: started, - lastActivityAtMs, - lastStdoutActivityAtMs, - lastStderrActivityAtMs, - lastProgressActivityAtMs, - }); - }, - }, - ); - - const hardTimer = setTimeout(() => { - timedOut = true; - timeoutKind = "hard"; - tree.terminate(); - }, timeoutMs); - hardTimer.unref(); - - const idleTimeoutMs = options.idleTimeoutMs; - const idleTimer = - idleTimeoutMs === undefined - ? undefined - : setInterval( - () => { - if (inFlightToolIds.size > 0) return; - if (Date.now() - lastActivityAtMs < idleTimeoutMs) return; - timedOut = true; - timeoutKind = "idle"; - tree.terminate(); - }, - Math.max(10, Math.min(1_000, Math.floor(idleTimeoutMs / 4))), - ); - idleTimer?.unref(); - }), - ); -} - -export function progressEventTime(line: string, receivedAtMs: number): number | undefined { - return streamEventActivity(line, receivedAtMs).progressAtMs; -} - -function streamEventActivity(line: string, receivedAtMs: number) { - const empty = { - progressAtMs: undefined as number | undefined, - startedToolIds: [] as string[], - completedToolIds: [] as string[], - }; - const trimmed = line.trim(); - if (!trimmed.startsWith("{")) return empty; - let event: any; - try { - event = JSON.parse(trimmed); - } catch { - return empty; - } - const content = Array.isArray(event?.message?.content) ? event.message.content : []; - const claudeStarts = content - .filter((block: any) => block?.type === "tool_use") - .map((block: any) => String(block.id ?? "claude:anonymous")); - const claudeCompletions = content - .filter((block: any) => block?.type === "tool_result") - .map((block: any) => String(block.tool_use_id ?? "claude:anonymous")); - const itemType = event?.item?.type ?? ""; - const codexTool = /command|file|mcp_tool|tool/.test(itemType); - const itemId = String(event?.item?.id ?? event?.item_id ?? "codex:anonymous"); - const codexStarts = event?.type === "item.started" && codexTool ? [itemId] : []; - const codexCompletions = - /^(?:item\.(?:completed|failed)|tool\.(?:completed|failed))$/.test(event?.type ?? "") && - codexTool - ? [itemId] - : []; - const part = event?.part; - const opencodeTool = part?.type === "tool"; - const opencodeStatus = String(part?.state?.status ?? part?.status ?? "").toLowerCase(); - const opencodeId = String(part?.callID ?? part?.id ?? part?.toolCallId ?? "opencode:anonymous"); - const opencodeStarts = - opencodeTool && /^(?:pending|running|started|in_progress)$/.test(opencodeStatus) - ? [opencodeId] - : []; - const opencodeCompletions = - opencodeTool && /^(?:completed|failed|error|cancelled)$/.test(opencodeStatus) - ? [opencodeId] - : []; - const piId = String(event?.toolCallId ?? "pi:anonymous"); - const piStarts = event?.type === "tool_execution_start" ? [piId] : []; - const piCompletions = event?.type === "tool_execution_end" ? [piId] : []; - const codexProgress = - /^(item\.(started|completed)|tool\.)/.test(event?.type ?? "") && - /command|file|mcp_tool|tool/.test(event?.item?.type ?? event?.type ?? ""); - const progress = - claudeStarts.length > 0 || - claudeCompletions.length > 0 || - codexCompletions.length > 0 || - codexProgress || - ["tool", "file", "patch"].includes(part?.type) || - /^tool_execution_(?:start|update|end)$/.test(event?.type ?? ""); - if (!progress) return empty; - - const raw = event.timestamp ?? event.time ?? event.created_at ?? event.createdAt; - let progressAtMs = receivedAtMs; - if (typeof raw === "number") progressAtMs = raw < 10_000_000_000 ? raw * 1_000 : raw; - if (typeof raw === "string") { - const parsed = Date.parse(raw); - if (Number.isFinite(parsed)) progressAtMs = parsed; - } - return { - progressAtMs, - startedToolIds: [...claudeStarts, ...codexStarts, ...opencodeStarts, ...piStarts], - completedToolIds: [ - ...claudeCompletions, - ...codexCompletions, - ...opencodeCompletions, - ...piCompletions, - ], - }; -} - -function formatSpawnError(error: unknown) { - if (error instanceof Error) return `${error.name}: ${error.message}`; - if (error && typeof error === "object" && "message" in error) { - const name = "name" in error && typeof error.name === "string" ? error.name : "Error"; - return `${name}: ${String(error.message)}`; - } - return String(error); -} - -export function spawnErrorCode(error: unknown): string | undefined { - if (!error || typeof error !== "object") return undefined; - for (const key of ["code", "reason"]) { - const value = (error as Record)[key]; - if (typeof value === "string" && value) return value; - } - const cause = (error as Record).cause; - return cause === error ? undefined : spawnErrorCode(cause); -} - -export function quoteArg(arg: string) { - return /^[a-zA-Z0-9_./:=@-]+$/.test(arg) ? arg : JSON.stringify(arg); -} - -export function tail(value: string, max = 4_000) { - return value.length <= max ? value : value.slice(-max); -} diff --git a/scripts/scaffbench/agents/index.ts b/scripts/scaffbench/agents/index.ts deleted file mode 100644 index 3fb366803..000000000 --- a/scripts/scaffbench/agents/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -export { providerForModel, agentLabelForModel } from "@scaffbench/agents/routing"; -export type { AgentProvider } from "@scaffbench/agents/routing"; -export { runClaude, claudeCostUsd, parseClaudeResult } from "@scaffbench/agents/claude"; -export { runCodex, codexCostUsd, parseCodexResult } from "@scaffbench/agents/codex"; -export { runAgy, parseAgyResult, agyModelString } from "@scaffbench/agents/agy"; -export { runOpencode, parseOpencodeResult } from "@scaffbench/agents/opencode"; -export { runKilo } from "@scaffbench/agents/kilo"; -export { - runPi, - parsePiResult, - piCommandArgs, - piProviderAndModel, - piThinkingArgs, -} from "@scaffbench/agents/pi"; -export { - agentRunCommandOptions, - runCommand, - quoteArg, - tail, - progressEventTime, - spawnErrorCode, -} from "@scaffbench/agents/command"; diff --git a/scripts/scaffbench/agents/kilo.ts b/scripts/scaffbench/agents/kilo.ts deleted file mode 100644 index 1b66e51e4..000000000 --- a/scripts/scaffbench/agents/kilo.ts +++ /dev/null @@ -1 +0,0 @@ -export { runOpencode as runKilo } from "@scaffbench/agents/opencode"; diff --git a/scripts/scaffbench/agents/opencode.ts b/scripts/scaffbench/agents/opencode.ts deleted file mode 100644 index 540d41f1d..000000000 --- a/scripts/scaffbench/agents/opencode.ts +++ /dev/null @@ -1,125 +0,0 @@ -import type { CommandExecutor } from "@effect/platform/CommandExecutor"; -import type { CommandResult, Effort } from "@scaffbench/types"; - -import * as FileSystem from "@effect/platform/FileSystem"; -import { agentRunCommandOptions, runCommand } from "@scaffbench/agents/command"; -import { bfSpec, GEN_TIMEOUT_MS } from "@scaffbench/constants"; -import * as Effect from "effect/Effect"; -import path from "node:path"; - -export function runOpencode(input: { - binary: "opencode" | "kilo"; - cwd: string; - prompt: string; - model: string; - effort: Effort; - useMcp: boolean; - bunx: string; - timeoutMs?: number; -}): Effect.Effect { - return Effect.gen(function* () { - if (input.useMcp) { - const fs = yield* FileSystem.FileSystem; - const config = { - mcp: { - "better-fullstack": { - type: "local", - command: [input.bunx, bfSpec("create-better-fullstack"), "mcp"], - enabled: true, - }, - }, - }; - yield* fs.writeFileString( - path.join(input.cwd, "opencode.json"), - `${JSON.stringify(config, null, 2)}\n`, - ); - } - const effortArgs = input.effort === "default" ? [] : ["--variant", input.effort]; - const modelId = input.model.replace(/^kilocode\//i, ""); - return yield* runCommand( - input.binary, - [ - "run", - "--format", - "json", - "--auto", - "--pure", - "-m", - modelId, - ...effortArgs, - "--dir", - input.cwd, - input.prompt, - ], - input.cwd, - input.timeoutMs ?? GEN_TIMEOUT_MS, - agentRunCommandOptions(input.binary), - ); - }); -} - -export function parseOpencodeResult(stdout: string): any | null { - let sessionId: string | undefined; - let outputTokens = 0; - let cost = 0; - let sawStep = false; - let sawTool = false; - let sawAssistantText = false; - let lastStepTokens = 0; - let stepReason: string | undefined; - let errorReason: string | undefined; - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (!trimmed.startsWith("{")) continue; - let event: any; - try { - event = JSON.parse(trimmed); - } catch { - continue; - } - if (typeof event?.sessionID === "string") sessionId = event.sessionID; - const part = event?.part; - if (part?.type === "tool") sawTool = true; - const assistantText = - part?.type === "text" - ? (part.text ?? part.content) - : event?.role === "assistant" || event?.message?.role === "assistant" - ? (event.text ?? event.content ?? event.message?.content) - : undefined; - if ( - (typeof assistantText === "string" && assistantText.trim().length > 0) || - ["refusal", "moderation"].includes(part?.type) - ) { - sawAssistantText = true; - } - if (part?.type === "step-finish" || part?.type === "step_finish") { - sawStep = true; - lastStepTokens = (part.tokens?.output ?? 0) + (part.tokens?.reasoning ?? 0); - outputTokens += lastStepTokens; - if (typeof part.cost === "number") cost += part.cost; - if (typeof part.reason === "string") stepReason = part.reason; - } - if (event?.type === "error" || part?.type === "error") { - const error = event?.error ?? part?.error ?? event?.message ?? part?.message; - errorReason = typeof error === "string" ? error : JSON.stringify(error ?? "unknown"); - } - } - if (!sawStep && sessionId === undefined && !errorReason) return null; - const terminalReason = errorReason - ? `error:${errorReason}` - : stepReason === "unknown" && outputTokens === 0 && !sawTool && !sawAssistantText - ? "opencode-unknown-zero-usage-no-tools" - : stepReason === "unknown" && lastStepTokens === 0 && sawTool && outputTokens > 0 - ? "opencode-unknown-zero-usage-step" - : stepReason; - return { - type: "result", - usage: sawStep ? { output_tokens: outputTokens } : undefined, - total_cost_usd: sawStep ? cost : undefined, - session_id: sessionId, - duration_ms: undefined, - terminal_reason: terminalReason, - tool_events: sawTool, - assistant_text: sawAssistantText, - }; -} diff --git a/scripts/scaffbench/agents/pi.ts b/scripts/scaffbench/agents/pi.ts deleted file mode 100644 index 8c6f9fb0b..000000000 --- a/scripts/scaffbench/agents/pi.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { CommandExecutor } from "@effect/platform/CommandExecutor"; -import type { CommandResult, Effort } from "@scaffbench/types"; -import type * as Effect from "effect/Effect"; - -import { agentRunCommandOptions, runCommand } from "@scaffbench/agents/command"; -import { GEN_TIMEOUT_MS } from "@scaffbench/constants"; - -export function piThinkingArgs(effort: Effort): string[] { - return effort === "default" ? [] : ["--thinking", effort]; -} - -export function piProviderAndModel(modelId: string): { provider: string; model: string } { - const unprefixed = modelId.replace(/^pi\//i, ""); - const separator = unprefixed.indexOf("/"); - if (separator <= 0 || separator === unprefixed.length - 1) { - throw new Error( - `Invalid Pi model id ${JSON.stringify(modelId)}; expected pi//`, - ); - } - return { - provider: unprefixed.slice(0, separator), - model: unprefixed.slice(separator + 1), - }; -} - -export function piCommandArgs(input: { prompt: string; model: string; effort: Effort }): string[] { - const resolved = piProviderAndModel(input.model); - return [ - "-p", - "--mode", - "json", - "--provider", - resolved.provider, - "--model", - resolved.model, - ...piThinkingArgs(input.effort), - "--no-session", - "--no-extensions", - "--no-context-files", - input.prompt, - ]; -} - -export function runPi(input: { - cwd: string; - prompt: string; - model: string; - effort: Effort; - timeoutMs?: number; -}): Effect.Effect { - return runCommand( - "pi", - piCommandArgs(input), - input.cwd, - input.timeoutMs ?? GEN_TIMEOUT_MS, - agentRunCommandOptions("pi"), - ); -} - -/** Parse Pi's JSONL stream, using completed assistant messages as the accounting - * boundary. turn_end and agent_end repeat those messages and must not be summed. */ -export function parsePiResult(stdout: string): any | null { - let sessionId: string | undefined; - let outputTokens = 0; - let totalCost = 0; - let sawUsage = false; - let sawCost = false; - let toolEvents = 0; - let sawAssistantText = false; - let terminalReason: string | undefined; - - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (!trimmed.startsWith("{")) continue; - let event: any; - try { - event = JSON.parse(trimmed); - } catch { - continue; - } - - if (event?.type === "session" && typeof event.id === "string") sessionId = event.id; - if (event?.type === "tool_execution_start") toolEvents += 1; - - if (event?.type === "message_end" && event?.message?.role === "assistant") { - const usage = event.message.usage; - if (usage && typeof usage === "object") { - outputTokens += numeric(usage.output) + numeric(usage.reasoning); - sawUsage = true; - if (typeof usage.cost?.total === "number") { - totalCost += usage.cost.total; - sawCost = true; - } - } - const content = Array.isArray(event.message.content) ? event.message.content : []; - if (content.some((block: any) => block?.type === "text" && block.text?.trim())) { - sawAssistantText = true; - } - const stopReason = event.message.stopReason; - if (typeof stopReason === "string" && !["stop", "toolUse"].includes(stopReason)) { - terminalReason = stopReason; - } - } - - if (event?.type === "error") { - const error = event.error ?? event.message ?? event.reason ?? "unknown"; - terminalReason = `error:${typeof error === "string" ? error : JSON.stringify(error)}`; - } - } - - if (!sessionId && !sawUsage && toolEvents === 0 && !terminalReason) return null; - if (!terminalReason && outputTokens === 0 && toolEvents === 0 && !sawAssistantText) { - terminalReason = "pi-zero-usage-no-tools"; - } - return { - type: "result", - usage: sawUsage ? { output_tokens: outputTokens } : undefined, - total_cost_usd: sawCost ? totalCost : undefined, - session_id: sessionId, - duration_ms: undefined, - terminal_reason: terminalReason, - tool_events: toolEvents, - assistant_text: sawAssistantText, - }; -} - -function numeric(value: unknown): number { - return typeof value === "number" && Number.isFinite(value) ? value : 0; -} diff --git a/scripts/scaffbench/agents/routing.ts b/scripts/scaffbench/agents/routing.ts deleted file mode 100644 index aacea5ad2..000000000 --- a/scripts/scaffbench/agents/routing.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type {} from "@scaffbench/types"; - -export type AgentProvider = "claude" | "codex" | "opencode" | "kilo" | "agy" | "pi"; - -/** Infer the agent that drives a model id by its prefix. opencode/* and kilo/* - * models are driven by the opencode/Kilo Code CLIs; GPT/o-series by Codex; Gemini - * by Google's Antigravity `agy` CLI (the standalone gemini CLI is sunset for - * individual tiers). */ -export function providerForModel(model: string): AgentProvider { - if (/^pi\//i.test(model)) return "pi"; - // `kilocode/` drives the Kilo binary with an arbitrary provider id - - // needed because bare `openai/*` ids are served by BOTH opencode and kilo - // (kilo picks them up via the user's OpenAI oauth): `kilocode/openai/x` - // disambiguates from opencode's `openai/x` and from kilo's own credit-gated - // `kilo/openai/x` catalog ids. The adapter strips the prefix before -m. - if (/^kilocode\//i.test(model)) return "kilo"; - if (/^kilo\//i.test(model)) return "kilo"; - // The opencode CLI serves several tiers: free (`opencode/*`), the paid "Go" - // subscription (`opencode-go/*`), and the Cloudflare AI Gateway passthrough - - // all route to the opencode adapter with the full id passed through unchanged. - if (/^(opencode(-go)?|cloudflare-ai-gateway|openai)\//i.test(model)) return "opencode"; - if (/gemini/i.test(model)) return "agy"; - if (/^(gpt|o\d|codex)/i.test(model)) return "codex"; - return "claude"; -} - -// Human label for the agent that drove a model - for summary.md headers. Derived -// from the model so non-Claude runs aren't mislabeled "Claude Code". -export function agentLabelForModel(model: string): string { - switch (providerForModel(model)) { - case "codex": - return "Codex"; - case "opencode": - return "opencode"; - case "kilo": - return "Kilo Code"; - case "agy": - return "Antigravity"; - case "pi": - return "Pi"; - default: - return "Claude Code"; - } -} diff --git a/scripts/scaffbench/calibrate.ts b/scripts/scaffbench/calibrate.ts deleted file mode 100644 index 98b10afcc..000000000 --- a/scripts/scaffbench/calibrate.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { RunOutcome, ScaffbenchOptions } from "@scaffbench/types"; - -import { CALIBRATION_WEAK_MODEL } from "@scaffbench/constants"; -import { rollupOutcome } from "@scaffbench/scoring"; -import path from "node:path"; - -export type CalibrationVerdict = "keep" | "cut" | "inconclusive"; - -export function calibrationOptions(options: ScaffbenchOptions) { - const common = { - ...options, - command: "run" as const, - repeats: 1, - paths: ["prompt" as const], - repair: false, - listSpecs: false, - writeMatrixOnly: false, - generateOnly: false, - validateExisting: false, - }; - return { - weak: { - ...common, - model: CALIBRATION_WEAK_MODEL, - efforts: ["default" as const], - outDir: path.join(options.outDir, "calibration", "weak"), - }, - strong: { - ...common, - model: options.model, - efforts: [options.efforts[0] ?? "default"], - outDir: path.join(options.outDir, "calibration", "strong"), - }, - }; -} - -/** Keep only discriminating specs: weak model fails and configured strong passes. */ -export function calibrationVerdict( - weak: RunOutcome | undefined, - strong: RunOutcome | undefined, -): CalibrationVerdict { - if (!weak || !strong) return "inconclusive"; - if ( - rollupOutcome(weak) === "infra-inconclusive" || - rollupOutcome(strong) === "infra-inconclusive" - ) { - return "inconclusive"; - } - return rollupOutcome(weak) === "model-failure" && strong === "success" ? "keep" : "cut"; -} - -export function formatCalibrationVerdict( - specId: string, - verdict: CalibrationVerdict, - weak: RunOutcome | undefined, - strong: RunOutcome | undefined, -) { - return `CALIBRATE ${specId}: ${verdict.toUpperCase()} weak=${weak ?? "missing"} strong=${strong ?? "missing"}`; -} diff --git a/scripts/scaffbench/cli.ts b/scripts/scaffbench/cli.ts deleted file mode 100644 index 71a0fc837..000000000 --- a/scripts/scaffbench/cli.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { ScaffbenchOptions } from "@scaffbench/types"; - -import { - CORE_SPEC_IDS, - CREATION_PATH_VALUES, - DEFAULT_EFFORTS, - DEFAULT_PATHS, - EFFORT_VALUES, -} from "@scaffbench/constants"; -import { SCAFFBENCH_2_SPECS } from "@scaffbench/specs"; -import path from "node:path"; - -export function parseList( - flag: string, - value: string | undefined, - allowed: readonly T[], - fallback: readonly T[], -) { - if (!value) return [...fallback]; - if (value === "all") return [...allowed]; - const items = value.split(",").map((item) => item.trim()); - const unknown = items.filter((item) => !allowed.includes(item as T)); - if (unknown.length > 0) { - throw new Error( - `--${flag}: unknown value${unknown.length === 1 ? "" : "s"} ${unknown.join(", ")}; allowed: ${allowed.join(", ")}`, - ); - } - return items as T[]; -} - -function parseBudget(value: string) { - const parsed = Number.parseFloat(value); - if (!/^\d+(?:\.\d+)?$/.test(value.trim()) || !Number.isFinite(parsed) || parsed < 0) { - throw new Error( - `--max-budget-usd: expected a non-negative number, got ${JSON.stringify(value)}`, - ); - } - return value; -} - -function parseRepeats(value: string | undefined) { - if (value === undefined) return 1; - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 1) { - throw new Error(`--repeats: expected a positive integer, got ${JSON.stringify(value)}`); - } - return parsed; -} - -function parseTopUp(value: string | undefined, repeatsGiven: boolean) { - if (value === undefined) return undefined; - const parsed = Number(value); - if (!Number.isInteger(parsed) || parsed < 2) { - throw new Error(`--top-up: expected an integer of at least 2, got ${JSON.stringify(value)}`); - } - if (repeatsGiven) { - throw new Error("--top-up extends an existing out-dir and cannot be combined with --repeats"); - } - return parsed; -} - -export function parseArgs(argv: string[]): ScaffbenchOptions { - const command = argv[0] === "calibrate" ? "calibrate" : "run"; - const args = new Map(); - for (let i = 0; i < argv.length; i += 1) { - const token = argv[i]; - if (!token || !token.startsWith("--")) continue; - const key = token.slice(2); - const next = argv[i + 1]; - if (next && !next.startsWith("--")) { - args.set(key, next); - i += 1; - } else { - args.set(key, "true"); - } - } - - const requestedOutDir = args.get("out-dir"); - const specIds = SCAFFBENCH_2_SPECS.map((spec) => spec.id); - const specsArg = args.get("specs") ?? args.get("spec"); - const specs = - specsArg === "core" || !specsArg - ? [...CORE_SPEC_IDS] - : parseList("specs", specsArg, specIds, CORE_SPEC_IDS); - const promptStyle = args.get("prompt-style") === "natural" ? "natural" : "explicit"; - const repeats = parseRepeats(args.get("repeats")); - const topUp = parseTopUp(args.get("top-up"), args.has("repeats")); - if (topUp !== undefined && !requestedOutDir) { - throw new Error("--top-up needs --out-dir pointing at the run to extend"); - } - if (topUp !== undefined && (args.has("validate-existing") || args.has("write-matrix-only"))) { - throw new Error( - "--top-up generates new trials and cannot be combined with --validate-existing or --write-matrix-only", - ); - } - - return { - command, - model: args.get("model") ?? "opus", - efforts: parseList("efforts", args.get("efforts"), EFFORT_VALUES, DEFAULT_EFFORTS), - paths: parseList("paths", args.get("paths"), CREATION_PATH_VALUES, DEFAULT_PATHS), - specs, - specsExplicit: args.has("specs") || args.has("spec"), - repeats, - topUp, - outDir: requestedOutDir - ? path.resolve(process.cwd(), requestedOutDir) - : path.resolve( - process.cwd(), - "testing/llm-benchmarks/v2", - new Date().toISOString().replace(/[-:]/g, "").replace(/\..+$/, "Z"), - ), - maxBudgetUsd: parseBudget(args.get("max-budget-usd") ?? "12"), - skipValidation: args.has("skip-validation"), - generateOnly: args.has("generate-only"), - validateExisting: args.has("validate-existing"), - forceRevalidate: args.has("force-revalidate"), - qualityGate: !args.has("no-quality-gate"), - noQualityGate: args.has("no-quality-gate"), - doctorCheck: args.has("doctor-check"), - routeCheck: args.has("route-check"), - promptStyle, - listSpecs: args.has("list-specs"), - writeMatrixOnly: args.has("write-matrix-only"), - repair: args.has("repair"), - }; -} - -export function selectedSpecs(specIds: readonly string[]) { - const requested = new Set(specIds); - return SCAFFBENCH_2_SPECS.filter((spec) => requested.has(spec.id)); -} diff --git a/scripts/scaffbench/code-metrics.ts b/scripts/scaffbench/code-metrics.ts deleted file mode 100644 index 137862cc7..000000000 --- a/scripts/scaffbench/code-metrics.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { CodeMetrics } from "@scaffbench/types"; - -import { PROJECT_WALK_SKIP_DIRECTORIES } from "@scaffbench/validation/shared"; -import { readdir, readFile } from "node:fs/promises"; -import path from "node:path"; - -const LOCKFILES = new Set([ - "bun.lock", - "bun.lockb", - "package-lock.json", - "yarn.lock", - "pnpm-lock.yaml", - "Cargo.lock", - "go.sum", - "mix.lock", - "poetry.lock", - "uv.lock", - "Pipfile.lock", - "packages.lock.json", - "composer.lock", - "Gemfile.lock", - "gradle.lockfile", -]); - -const BINARY_EXTENSIONS = new Set([ - "png", - "jpg", - "jpeg", - "gif", - "webp", - "ico", - "pdf", - "woff", - "woff2", - "ttf", - "otf", - "eot", - "zip", - "jar", - "wasm", - "keystore", - "p8", - "p12", - "db", - "sqlite", -]); - -const BINARY_SNIFF_BYTES = 8 * 1024; - -/** Measure authored project volume before validation installs or builds anything. */ -export async function measureProjectCode(dir: string): Promise { - const metrics: CodeMetrics = { files: 0, lines: 0, bytes: 0 }; - - async function visit(currentDir: string): Promise { - const entries = await readdir(currentDir, { withFileTypes: true }); - for (const entry of entries) { - if (entry.isDirectory()) { - if (!PROJECT_WALK_SKIP_DIRECTORIES.has(entry.name)) { - await visit(path.join(currentDir, entry.name)); - } - continue; - } - if (!entry.isFile() || LOCKFILES.has(entry.name)) continue; - - const extension = path.extname(entry.name).slice(1).toLowerCase(); - if (BINARY_EXTENSIONS.has(extension)) continue; - - const contents = await readFile(path.join(currentDir, entry.name)); - if (contents.subarray(0, BINARY_SNIFF_BYTES).includes(0)) continue; - - metrics.files += 1; - metrics.bytes += contents.byteLength; - for (const byte of contents) { - if (byte === 0x0a) metrics.lines += 1; - } - if (contents.byteLength > 0 && contents[contents.byteLength - 1] !== 0x0a) { - metrics.lines += 1; - } - } - } - - await visit(dir); - return metrics; -} diff --git a/scripts/scaffbench/constants.ts b/scripts/scaffbench/constants.ts deleted file mode 100644 index 955ca0052..000000000 --- a/scripts/scaffbench/constants.ts +++ /dev/null @@ -1,245 +0,0 @@ -import type { BenchmarkSpec, CreationPath, Effort } from "@scaffbench/types"; - -import { runCommand } from "@scaffbench/agents/command"; -import * as Effect from "effect/Effect"; - -export const HARNESS_VERSION = "3.1.0"; -export const SCAFFBENCH_SUITE_VERSION = "3.0"; -export const PROMPT_VERSION = "2026-08-21-scaffbench-3.1"; -export const MIN_RANKED_TRIALS = 1; -export const MIN_CI_RUNS = 8; - -export const VALIDATION_RESOURCE_PROFILE_ID = "low-2w-v1"; -export const VALIDATION_RESOURCE_ENV: Readonly> = { - GOMAXPROCS: "2", - CARGO_BUILD_JOBS: "2", - UV_CONCURRENT_BUILDS: "2", - UV_CONCURRENT_INSTALLS: "2", - UV_CONCURRENT_DOWNLOADS: "8", - ERL_FLAGS: "+S 2:2", - JAVA_TOOL_OPTIONS: "-XX:ActiveProcessorCount=2", - MSBUILDDISABLENODEREUSE: "1", - GIT_TERMINAL_PROMPT: "0", -}; -export const VALIDATION_ENV_SCRUB_PATTERN = - /(TOKEN|SECRET|PASSWORD|CREDENTIAL|API_KEY|APIKEY|PRIVATE_KEY|AUTH|SSH_|AWS_|GCP_|GOOGLE_APPLICATION|AZURE_|OPENAI|ANTHROPIC|GEMINI|GH_|GITHUB_|NPM_|VERCEL|CLOUDFLARE|SENTRY|POSTHOG|STRIPE|SUPABASE|DATABASE_URL)/i; -export const VALIDATION_OUTPUT_LIMIT_BYTES = 16 * 1024 * 1024; - -export const SCAFFBENCH_SPEC_SCORE_WEIGHTS = { core: 0.6, quality: 0.2, stack: 0.2 } as const; -export const VALIDATION_CACHE_VERSION = 9; - -let RESOLVED_BF_VERSION = "latest"; - -export function setResolvedBfVersion(version: string) { - RESOLVED_BF_VERSION = version; -} - -export function resolvedBfVersion() { - return RESOLVED_BF_VERSION; -} - -export function bfSpec(pkg: "better-fullstack" | "create-better-fullstack") { - return `${pkg}@${RESOLVED_BF_VERSION}`; -} - -export function resolveBfVersion() { - return Effect.gen(function* () { - const version = yield* tryCommandText( - "npm", - ["view", "create-better-fullstack@latest", "version"], - process.cwd(), - ); - return version && /^\d+\.\d+\.\d+/.test(version) ? version : "latest"; - }); -} -export const EFFORT_VALUES: readonly Effort[] = [ - "default", - "low", - "medium", - "high", - "xhigh", - "max", -]; -export const CREATION_PATH_VALUES: readonly CreationPath[] = ["prompt", "mcp"]; -export const DEFAULT_EFFORTS: readonly Effort[] = ["default"]; -export const DEFAULT_PATHS: readonly CreationPath[] = ["prompt"]; - -export function resolveSpecPaths( - spec: BenchmarkSpec, - requested: readonly CreationPath[], -): CreationPath[] { - const allowed: readonly CreationPath[] = - spec.paths ?? (spec.supportedByBetterFullstack === false ? ["prompt"] : requested); - return requested.filter((path) => allowed.includes(path)); -} - -export const GEN_TIMEOUT_MS = 90 * 60_000; -/** @deprecated Use GEN_TIMEOUT_MS. */ -export const CLAUDE_TIMEOUT_MS = GEN_TIMEOUT_MS; -export const GEN_IDLE_TIMEOUT_MS = 20 * 60_000; -export const TIMEOUT_PROGRESS_WINDOW_MS = 10 * 60_000; -export const VALIDATION_TIMEOUT_MS = 20 * 60_000; -export const VALIDATION_PROJECT_TIMEOUT_MS = 90 * 60_000; -export const VALIDATION_ROOT_CAP = 12; -export const ESTIMATED_BUDGET_TOLERANCE = 1.25; -export const FAST_TIMEOUT_MS = 60_000; -export const QUEUE_POLL_MS = 5_000; -export const STALE_LOCK_MS = 6 * 60 * 60_000; -export const CALIBRATION_WEAK_MODEL = "opencode/deepseek-v4-flash-free"; - -export function generationTimeoutMs(spec: Pick) { - const multiplier = spec.timeoutMultiplier ?? 1; - return GEN_TIMEOUT_MS * (Number.isFinite(multiplier) && multiplier > 0 ? multiplier : 1); -} -export const CORE_SPEC_IDS = [ - "ai-search-workbench", - "rust-leptos-axum", - "python-ingestion-api", - "go-realtime-api", - "multi-dotnet-ops", - "ts-svelte-edge-orpc", - "dotnet-blazor-cqrs", - "multi-ts-go-grpc", - "java-spring-jooq-keycloak", - "elixir-broadway-absinthe", - "react-native-expo", - "frontier-polyglot-proto", - "frontier-effect-eventsourcing", -] as const; - -export const AI_SEARCH_STACK = { - frontend: "tanstack-router", - backend: "hono", - runtime: "bun", - api: "orpc", - database: "postgres", - orm: "drizzle", - auth: "better-auth", - ai: "vercel-ai", - vectorDb: "qdrant", - search: "opensearch", - jobQueue: "inngest", - logging: "pino", - observability: "opentelemetry", - stateManagement: "tanstack-store", - forms: "tanstack-form", - validation: "valibot", - testing: "vitest-playwright", - i18n: "paraglide", - cssFramework: "tailwind", - uiLibrary: "shadcn-ui", -} as const; - -export const AI_SEARCH_ADDONS = ["vite-plus", "devcontainer", "github-actions"] as const; - -export const AI_SEARCH_FLAGS = [ - "--ecosystem", - "typescript", - "--frontend", - "tanstack-router", - "--backend", - "hono", - "--runtime", - "bun", - "--api", - "orpc", - "--database", - "postgres", - "--orm", - "drizzle", - "--db-setup", - "none", - "--auth", - "better-auth", - "--payments", - "none", - "--email", - "none", - "--file-upload", - "none", - "--logging", - "pino", - "--observability", - "opentelemetry", - "--feature-flags", - "none", - "--analytics", - "none", - "--effect", - "none", - "--state-management", - "tanstack-store", - "--forms", - "tanstack-form", - "--validation", - "valibot", - "--testing", - "vitest-playwright", - "--ai", - "vercel-ai", - "--realtime", - "none", - "--job-queue", - "inngest", - "--animation", - "none", - "--css-framework", - "tailwind", - "--ui-library", - "shadcn-ui", - "--cms", - "none", - "--caching", - "none", - "--rate-limit", - "none", - "--i18n", - "paraglide", - "--search", - "opensearch", - "--vector-db", - "qdrant", - "--file-storage", - "none", - "--web-deploy", - "none", - "--server-deploy", - "none", - "--addons", - "vite-plus", - "devcontainer", - "github-actions", - "--examples", - "none", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--shadcn-base", - "radix", - "--shadcn-style", - "nova", - "--shadcn-icon-library", - "lucide", - "--shadcn-color-theme", - "neutral", - "--shadcn-base-color", - "neutral", - "--shadcn-font", - "inter", - "--shadcn-radius", - "default", - "--no-install", - "--no-git", - "--disable-analytics", -] as const; - -export function tryCommandText(command: string, args: readonly string[], cwd: string) { - return runCommand(command, args, cwd, FAST_TIMEOUT_MS).pipe( - Effect.map((result) => { - if (result.exitCode !== 0) return undefined; - return result.stdout.trim(); - }), - Effect.catchAll(() => Effect.succeed(undefined)), - ); -} diff --git a/scripts/scaffbench/index.ts b/scripts/scaffbench/index.ts deleted file mode 100644 index ed4bdbf20..000000000 --- a/scripts/scaffbench/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export * from "@scaffbench/types"; -export * from "@scaffbench/constants"; -export * from "@scaffbench/specs"; -export * from "@scaffbench/cli"; -export * from "@scaffbench/prompts"; -export * from "@scaffbench/agents"; -export * from "@scaffbench/validation"; -export * from "@scaffbench/validation/cache"; -export * from "@scaffbench/validation/classification"; -export * from "@scaffbench/scoring"; -export * from "@scaffbench/code-metrics"; -export * from "@scaffbench/summary"; -export * from "@scaffbench/runner"; -export * from "@scaffbench/calibrate"; diff --git a/scripts/scaffbench/process-tree.ts b/scripts/scaffbench/process-tree.ts deleted file mode 100644 index 28837c90d..000000000 --- a/scripts/scaffbench/process-tree.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { spawn } from "node:child_process"; - -type SpawnedProcess = { - pid?: number; - stdout: { on(event: "data", listener: (chunk: Buffer) => void): void } | null; - stderr: { on(event: "data", listener: (chunk: Buffer) => void): void } | null; - on(event: "error", listener: (cause: Error & { code?: string }) => void): void; - on(event: "close", listener: (code: number | null) => void): void; -}; - -export const KILL_ESCALATION_MS = 3_000; - -export function killProcessGroup(pid: number, signal: NodeJS.Signals) { - try { - process.kill(-pid, signal); - } catch {} -} - -export type ProcessTree = { - pid?: number; - terminate(): void; - kill(): void; -}; - -export function spawnProcessTree( - command: string, - args: readonly string[], - options: { cwd: string; env: NodeJS.ProcessEnv }, - handlers: { - onStdout?: (chunk: Buffer) => void; - onStderr?: (chunk: Buffer) => void; - onError: (cause: Error & { code?: string }) => void; - onClose: (code: number | null) => void; - }, -): ProcessTree { - const child = spawn(command, [...args], { - cwd: options.cwd, - env: options.env, - detached: true, - stdio: ["ignore", "pipe", "pipe"], - }) as unknown as SpawnedProcess; - - if (handlers.onStdout) child.stdout?.on("data", handlers.onStdout); - if (handlers.onStderr) child.stderr?.on("data", handlers.onStderr); - child.on("error", handlers.onError); - child.on("close", handlers.onClose); - - return { - get pid() { - return child.pid; - }, - terminate() { - if (child.pid === undefined) return; - killProcessGroup(child.pid, "SIGTERM"); - setTimeout(() => { - if (child.pid !== undefined) killProcessGroup(child.pid, "SIGKILL"); - }, KILL_ESCALATION_MS).unref(); - }, - kill() { - if (child.pid !== undefined) killProcessGroup(child.pid, "SIGKILL"); - }, - }; -} diff --git a/scripts/scaffbench/prompts.ts b/scripts/scaffbench/prompts.ts deleted file mode 100644 index 23d0f2bde..000000000 --- a/scripts/scaffbench/prompts.ts +++ /dev/null @@ -1,57 +0,0 @@ -import type { BenchmarkSpec, CreationPath, PromptStyle } from "@scaffbench/types"; - -import { quoteArg } from "@scaffbench/agents/command"; -import { bfSpec } from "@scaffbench/constants"; - -export function canonicalCommand(spec: BenchmarkSpec, projectName: string) { - return ["bun", "create", bfSpec("better-fullstack"), projectName, ...spec.canonicalFlags] - .map((part) => quoteArg(part)) - .join(" "); -} - -export function promptFor( - spec: BenchmarkSpec, - pathMode: CreationPath, - runDir: string, - projectName: string, - promptStyle: PromptStyle, -) { - const body = - promptStyle === "natural" - ? spec.naturalPrompt - : `Benchmark target: ${spec.title} -Requirements: -${spec.requirements.map((requirement) => `- ${requirement}`).join("\n")}`; - - const discoveryLane = promptStyle === "natural" && spec.acceptanceSets !== undefined; - const libraryGuidance = discoveryLane - ? "" - : ` - -Important scoring rule: choosing the right library matters. -${spec.rightLibraryNotes.map((note) => `- ${note}`).join("\n")}`; - - const base = `You are running in an empty benchmark workspace: -${runDir} - -Create exactly one project directory named \`${projectName}\`. -Do not ask questions. Do not write outside the current working directory. Do not initialize git. -You may install dependencies, query package registries, run builds or type checks, and start servers to verify your work before finishing. Kill every process you start; nothing may still be running when you finish. -At the end, report the commands you ran and any errors you hit. - -${body}${libraryGuidance}`; - - if (pathMode === "prompt") { - return `${base} - -Creation mode: prompt-only. -Do not use the Better-Fullstack MCP server, Better-Fullstack CLI, Better-Fullstack website, or files from the Better-Fullstack repository. -Create the project from scratch by writing the files and manifests needed for a runnable starter.`; - } - - return `${base} - -Creation mode: Better-Fullstack MCP. -Use the Better-Fullstack MCP tools. Start with bfs_get_guidance, then use schema/compatibility/plan as needed, and call bfs_create_project to create the project. -Do not use the Better-Fullstack CLI for creation.`; -} diff --git a/scripts/scaffbench/runner.ts b/scripts/scaffbench/runner.ts deleted file mode 100644 index da59a1890..000000000 --- a/scripts/scaffbench/runner.ts +++ /dev/null @@ -1,1283 +0,0 @@ -import type { - BenchmarkSpec, - CreationPath, - Effort, - ProjectValidation, - RepairResult, - RunProtocol, - TopUpRecord, - RunResult, - ScaffbenchOptions, - StepResult, -} from "@scaffbench/types"; - -import * as FileSystem from "@effect/platform/FileSystem"; -import { - agyModelString, - claudeCostUsd, - parseAgyResult, - parseClaudeResult, - parseCodexResult, - parseOpencodeResult, - parsePiResult, - providerForModel, - runAgy, - runClaude, - runCodex, - runOpencode, - runPi, - tail, -} from "@scaffbench/agents"; -import { - calibrationOptions, - calibrationVerdict, - formatCalibrationVerdict, -} from "@scaffbench/calibrate"; -import { selectedSpecs } from "@scaffbench/cli"; -import { measureProjectCode } from "@scaffbench/code-metrics"; -import { - CREATION_PATH_VALUES, - EFFORT_VALUES, - HARNESS_VERSION, - PROMPT_VERSION, - SCAFFBENCH_SUITE_VERSION, - VALIDATION_CACHE_VERSION, - VALIDATION_RESOURCE_PROFILE_ID, - QUEUE_POLL_MS, - STALE_LOCK_MS, - bfSpec, - resolveBfVersion, - resolveSpecPaths, - setResolvedBfVersion, - generationTimeoutMs, -} from "@scaffbench/constants"; -import { canonicalCommand, promptFor } from "@scaffbench/prompts"; -import { - deriveFailureTags, - emptyAcceptanceScore, - emptyArtifactScore, - scoreProject, - scoreToolCompliance, - validationPassed, - classifyOutcome, - outcomeEvidenceFor, - rollupOutcome, - stepBaseName, -} from "@scaffbench/scoring"; -import { SCAFFBENCH_2_SPECS } from "@scaffbench/specs"; -import { collectMetadata, effectiveReasoning, writeSummary } from "@scaffbench/summary"; -import { archiveProjectSource, findProjectDir } from "@scaffbench/validation"; -import { validateProjectCached } from "@scaffbench/validation/cache"; -import * as Effect from "effect/Effect"; -import * as Either from "effect/Either"; -import * as Option from "effect/Option"; -import { createHash } from "node:crypto"; -import os from "node:os"; -import path from "node:path"; - -type Log = (message: string) => void; - -function fromPromise(evaluate: () => Promise) { - return Effect.tryPromise({ try: evaluate, catch: (cause) => cause }); -} - -export function runScaffbench(options: ScaffbenchOptions, log: Log = console.log) { - const program = - options.command === "calibrate" - ? runCalibrationUnlocked(options, log) - : runScaffbenchUnlocked(options, log); - if (options.listSpecs || options.writeMatrixOnly) return program; - return withScaffbenchQueue(options, log, program); -} - -function runCalibrationUnlocked(options: ScaffbenchOptions, log: Log) { - return Effect.gen(function* () { - const specs = selectedSpecs(options.specs); - if (specs.length !== 1) { - return yield* Effect.fail(new Error("scaffbench calibrate requires exactly one --spec ")); - } - const spec = specs[0]!; - const calibration = calibrationOptions(options); - log(`CALIBRATE ${spec.id}: weak=${calibration.weak.model}`); - yield* runScaffbenchUnlocked(calibration.weak, log); - log(`CALIBRATE ${spec.id}: strong=${calibration.strong.model}`); - yield* runScaffbenchUnlocked(calibration.strong, log); - const weakResults = yield* readExistingResults(calibration.weak.outDir); - const strongResults = yield* readExistingResults(calibration.strong.outDir); - const weak = weakResults.find((result) => result.specId === spec.id); - const strong = strongResults.find((result) => result.specId === spec.id); - const weakOutcome = weak ? classifyOutcome(weak) : undefined; - const strongOutcome = strong ? classifyOutcome(strong) : undefined; - log( - formatCalibrationVerdict( - spec.id, - calibrationVerdict(weakOutcome, strongOutcome), - weakOutcome, - strongOutcome, - ), - ); - }); -} - -function runScaffbenchUnlocked(options: ScaffbenchOptions, log: Log) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - if (options.generateOnly && options.validateExisting) { - return yield* Effect.fail( - new Error("--generate-only and --validate-existing cannot be used together"), - ); - } - - if (options.listSpecs) { - const listed = selectedSpecs(options.specs); - for (const spec of listed.length ? listed : SCAFFBENCH_2_SPECS) { - log(`${spec.id}\t${spec.lane}\t${spec.family}\t${spec.title}`); - } - return; - } - - yield* fs.makeDirectory(options.outDir, { recursive: true }); - const existingSummary = yield* readExistingSummary(options.outDir); - const recordedProtocol = recordedRunProtocol(existingSummary); - if (options.topUp !== undefined && (options.validateExisting || options.writeMatrixOnly)) { - return yield* Effect.fail( - new Error( - "--top-up generates new trials and cannot run in validate-only or matrix-only mode", - ), - ); - } - if (options.topUp !== undefined && recordedProtocol === undefined) { - return yield* Effect.fail( - new Error( - `--top-up extends a finished run, but ${options.outDir} has no recorded run protocol`, - ), - ); - } - const runOptions: ScaffbenchOptions = - options.validateExisting || options.topUp !== undefined - ? { ...options, ...recordedRunOptions(existingSummary) } - : options; - - const specs = selectedSpecs(runOptions.specs); - const specOrderSeed = specShuffleSeed(runOptions); - const topUpSpecIds = - options.topUp === undefined - ? undefined - : topUpSpecSelection( - options.specsExplicit ? options.specs : runOptions.specs, - runOptions.specs, - ); - const previousTopUps = recordedProtocol?.topUps ?? []; - const topUps = - topUpSpecIds && options.topUp !== undefined - ? [ - ...previousTopUps, - { trials: options.topUp, specs: topUpSpecIds, recordedAt: new Date().toISOString() }, - ] - : previousTopUps; - const runProtocol = { - repeats: runOptions.repeats, - seed: specOrderSeed, - ...(topUps.length > 0 ? { topUps } : {}), - } satisfies RunProtocol; - const results = completedResults(existingSummary); - const recordedResults = recordedRunResults(existingSummary); - const schedule = topUpSpecIds - ? buildGenerationSchedule( - selectedSpecs(topUpSpecIds), - { ...runOptions, repeats: options.topUp! }, - specOrderSeed, - ) - : buildGenerationSchedule(specs, runOptions, specOrderSeed); - if (topUpSpecIds) { - const pending = schedule.filter( - (entry) => - !findCompletedTrial( - results, - entry.spec, - runOptions.model, - entry.effort, - entry.pathMode, - entry.trial, - ), - ); - if (pending.length === 0) { - return yield* Effect.fail( - new Error( - `--top-up ${options.topUp}: every selected spec already has ${options.topUp} completed trials`, - ), - ); - } - log( - `TOP-UP ${runOptions.model}: ${pending.length} generation(s) to reach ${options.topUp} trials on ${topUpSpecIds.join(", ")}`, - ); - } - if (schedule.length === 0 && !runOptions.writeMatrixOnly) { - return yield* Effect.fail( - new Error( - `empty schedule: specs=${runOptions.specs.join(",") || "(none)"} ` + - `efforts=${runOptions.efforts.join(",")} paths=${runOptions.paths.join(",")}`, - ), - ); - } - if (recordedResults.length > 0 && !options.validateExisting) { - assertResumeProtocol({ - recorded: recordedRunProtocol(existingSummary), - current: runProtocol, - results: recordedResults, - schedule, - model: runOptions.model, - }); - } - setResolvedBfVersion(yield* resolveBfVersion()); - yield* writeHarnessFiles(runOptions.outDir, runOptions, specs); - const metadata: Record = yield* collectMetadata(runOptions); - metadata.specOrderSeed = specOrderSeed; - metadata.runProtocol = runProtocol; - - if (options.writeMatrixOnly) { - yield* writeSummaryEffect(runOptions.outDir, [], runOptions, specs, metadata); - log(`Wrote ScaffBench 2 matrix to ${runOptions.outDir}`); - return; - } - - const home = process.env.HOME ?? ""; - const nativeBunx = path.join(home, ".bun", "bin", "bunx"); - const bunx = (yield* fs.exists(nativeBunx)) ? nativeBunx : "bunx"; - const emptyMcpPath = path.join(runOptions.outDir, "empty-mcp.json"); - const bfsMcpPath = path.join(runOptions.outDir, "better-fullstack-mcp.json"); - yield* writeMcpConfigs(emptyMcpPath, bfsMcpPath, bunx); - - const provider = providerForModel(runOptions.model); - assertScheduleSupported(schedule, provider, runOptions.model); - const workspaceRoot = path.join( - os.tmpdir(), - "scaffbench21-work", - path.basename(runOptions.outDir), - ); - yield* fs.makeDirectory(workspaceRoot, { recursive: true }); - - if (!options.validateExisting) { - for (const spec of specs) { - const specPaths = resolveSpecPaths(spec, runOptions.paths); - const skippedPaths = runOptions.paths.filter((pathMode) => !specPaths.includes(pathMode)); - if (skippedPaths.length > 0) { - log( - `PATHS ${spec.id}: runs ${specPaths.join(", ") || "(none)"}, skipping ${skippedPaths.join(", ")} (frontier/prompt-only or pinned spec.paths)`, - ); - } - } - - yield* Effect.forEach( - schedule, - ({ spec, effort, pathMode, trial }) => - runOneGeneration({ - spec, - effort, - pathMode, - trial, - options: runOptions, - specs, - provider, - bunx, - emptyMcpPath, - bfsMcpPath, - workspaceRoot, - results, - metadata, - specOrderSeed, - log, - }), - { concurrency: 1, discard: true }, - ); - } - - if (!options.skipValidation && !options.generateOnly) { - yield* validatePendingResults(results, runOptions, specs, metadata, log); - if (options.repair) { - yield* repairFailedResults({ - results, - options: runOptions, - specs, - metadata, - provider, - bunx, - emptyMcpPath, - log, - }); - } - } else if (options.generateOnly) { - log("Generation finished; validation deferred. Re-run the same out-dir to validate."); - } - }); -} - -function runOneGeneration(input: { - spec: BenchmarkSpec; - effort: Effort; - pathMode: CreationPath; - trial: number; - options: ScaffbenchOptions; - specs: readonly BenchmarkSpec[]; - provider: ReturnType; - bunx: string; - emptyMcpPath: string; - bfsMcpPath: string; - workspaceRoot: string; - results: RunResult[]; - metadata: Record; - specOrderSeed: number; - log: Log; -}) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const { effort, log, options, pathMode, provider, results, spec, trial } = input; - const configuredTrials = Math.max(options.repeats, options.topUp ?? 0); - const projectName = buildProjectName(spec, pathMode, effort, trial, configuredTrials); - const id = buildRunId(spec, options.model, effort, pathMode, trial, configuredTrials); - const runDir = path.join(options.outDir, "runs", id); - const workDir = path.join(input.workspaceRoot, id); - - const existing = findCompletedTrial(results, spec, options.model, effort, pathMode, trial); - if (existing) { - log(`SKIP ${existing.id} already present`); - return; - } - - yield* fs.makeDirectory(runDir, { recursive: true }); - - yield* fs.remove(workDir, { recursive: true, force: true }); - yield* fs.makeDirectory(workDir, { recursive: true }); - - const prompt = promptFor(spec, pathMode, workDir, projectName, options.promptStyle); - yield* fs.writeFileString(path.join(runDir, "prompt.txt"), prompt); - yield* fs.writeFileString( - path.join(runDir, "canonical-command.txt"), - `${canonicalCommand(spec, projectName)}\n`, - ); - - log(`RUN ${id}`); - const started = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - const timeoutMs = generationTimeoutMs(spec); - const agentResult = - provider === "codex" - ? yield* runCodex({ - cwd: workDir, - prompt, - model: options.model, - effort, - useMcp: pathMode === "mcp", - bunx: input.bunx, - timeoutMs, - }) - : provider === "agy" - ? yield* runAgy({ cwd: workDir, prompt, model: options.model, effort, timeoutMs }) - : provider === "pi" - ? yield* runPi({ cwd: workDir, prompt, model: options.model, effort, timeoutMs }) - : provider === "opencode" || provider === "kilo" - ? yield* runOpencode({ - binary: provider, - cwd: workDir, - prompt, - model: options.model, - effort, - useMcp: pathMode === "mcp", - bunx: input.bunx, - timeoutMs, - }) - : yield* runClaude({ - cwd: workDir, - prompt, - model: options.model, - effort, - maxBudgetUsd: options.maxBudgetUsd, - mcpConfig: pathMode === "mcp" ? input.bfsMcpPath : input.emptyMcpPath, - timeoutMs, - }); - const finished = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - const durationMs = finished - started; - - yield* fs.writeFileString(path.join(runDir, "claude.stdout.json"), agentResult.stdout); - yield* fs.writeFileString(path.join(runDir, "claude.stderr.log"), agentResult.stderr); - - const parsed = - provider === "codex" - ? parseCodexResult(agentResult.stdout, options.model) - : provider === "agy" - ? parseAgyResult(agentResult.stdout) - : provider === "pi" - ? parsePiResult(agentResult.stdout) - : provider === "opencode" || provider === "kilo" - ? parseOpencodeResult(agentResult.stdout) - : parseClaudeResult(agentResult.stdout); - const generatedDir = yield* fromPromise(() => findProjectDir(workDir, projectName)); - const validation = options.skipValidation - ? { - projectExists: generatedDir !== null, - qualityGateRequested: qualityGateRequested(options), - skipped: true, - steps: {}, - } - : deferredValidation(generatedDir !== null, options); - const scored = generatedDir - ? yield* fromPromise(() => scoreProject(spec, generatedDir, options.promptStyle)) - : { - artifact: emptyArtifactScore(spec, options.promptStyle), - faithfulness: undefined, - acceptance: - options.promptStyle === "natural" && spec.acceptanceSets - ? emptyAcceptanceScore(spec) - : undefined, - }; - const codeMetrics = generatedDir - ? yield* fromPromise(() => measureProjectCode(generatedDir)) - : undefined; - const toolCompliance = yield* fromPromise(() => - scoreToolCompliance(pathMode, generatedDir, agentResult), - ); - - let projectDir = generatedDir; - if (generatedDir) { - const archivedDir = path.join(runDir, projectName); - const archive = yield* Effect.either( - fromPromise(() => archiveProjectSource(generatedDir, archivedDir)), - ); - if (Either.isRight(archive)) projectDir = archivedDir; - else { - const error = archive.left; - log( - `WARN archive failed for ${id}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - } - if (!generatedDir || projectDir !== generatedDir) { - yield* fs.remove(workDir, { recursive: true, force: true }); - } - - const totalCostUsd = resolvedCostUsd(provider, options.model, parsed); - const maxBudgetUsd = normalizedBudget(options.maxBudgetUsd); - const result: RunResult = { - id, - specId: spec.id, - specTitle: spec.title, - model: options.model, - effort, - effectiveReasoning: effectiveReasoning(options.model, effort), - path: pathMode, - trial, - promptStyle: options.promptStyle, - runDir, - projectName, - projectDir, - codeMetrics, - claude: { - exitCode: agentResult.exitCode, - timedOut: agentResult.timedOut, - durationMs, - resultDurationMs: parsed?.duration_ms, - outputTokens: parsed?.usage?.output_tokens, - totalCostUsd, - sessionId: parsed?.session_id, - terminalReason: parsed?.terminal_reason, - spawnError: agentResult.spawnError, - spawnErrorCode: agentResult.spawnErrorCode, - timeoutKind: agentResult.timeoutKind, - timeoutProgress: agentResult.timeoutProgress, - stderrTail: agentResult.stderrTail, - }, - budgetPolicy: { - budgetEnforced: provider === "claude", - maxBudgetUsd, - }, - provenance: { - suiteVersion: SCAFFBENCH_SUITE_VERSION, - harnessVersion: HARNESS_VERSION, - validationCacheVersion: VALIDATION_CACHE_VERSION, - promptVersion: PROMPT_VERSION, - resourceProfileId: VALIDATION_RESOURCE_PROFILE_ID, - agentAdapter: provider, - configuredTrials, - specOrderSeed: input.specOrderSeed, - }, - validation, - stackScore: scored.artifact, - generatorFaithfulness: scored.faithfulness, - acceptanceScore: scored.acceptance, - toolCompliance, - failureTags: [], - }; - result.outcome = classifyOutcome(result); - result.outcomeEvidence = outcomeEvidenceFor(result); - result.failureTags = deriveFailureTags(result); - results.push(result); - yield* writeSummaryEffect(options.outDir, results, options, input.specs, input.metadata); - log( - `DONE ${id} exit=${result.claude.exitCode} validation=${ - result.validation.deferred ? "deferred" : validationPassed(result) - } stack=${result.stackScore.matched}/${result.stackScore.total}`, - ); - }); -} - -export function selectRepairFailure(result: RunResult): [string, StepResult] | undefined { - return Object.entries(result.validation.steps).find((entry): entry is [string, StepResult] => { - const [name, step] = entry; - if ( - !step || - ["lint", "format", "test", "doctor", "route", "tidy"].includes(stepBaseName(name)) - ) { - return false; - } - return ( - step.status === "skip" || step.exitCode !== 0 || step.timedOut || step.spawnError === true - ); - }); -} - -export function repairPromptFor( - stepName: string, - step: NonNullable, -) { - const diagnostics = (step.stderrTail || step.stdoutTail || "No diagnostic output was captured.") - .split("\n") - .slice(-50) - .join("\n"); - return `Repair the existing project in the current directory. Do not recreate it. Kill every process you start before finishing. -The ScaffBench validation step \`${stepName}\` failed. Make the smallest changes needed for that step and the project build to pass, then run a focused check. - -Failing-step stderr tail: -${diagnostics}`; -} - -function repairFailedResults(input: { - results: RunResult[]; - options: ScaffbenchOptions; - specs: readonly BenchmarkSpec[]; - metadata: Record; - provider: ReturnType; - bunx: string; - emptyMcpPath: string; - log: Log; -}) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const specsById = new Map(input.specs.map((spec) => [spec.id, spec])); - const failed = input.results.filter( - (result) => - !result.repair && - result.projectDir && - rollupOutcome(classifyOutcome(result)) === "model-failure" && - selectRepairFailure(result), - ); - if (failed.length === 0) return; - input.log(`REPAIR ${failed.length} failed cell${failed.length === 1 ? "" : "s"}`); - - yield* Effect.forEach( - failed, - (result) => - Effect.gen(function* () { - const spec = specsById.get(result.specId); - const failure = selectRepairFailure(result); - if (!spec || !failure || !result.projectDir) return; - const [stepName, step] = failure; - const repairProvider = providerForModel(result.model); - const prompt = repairPromptFor(stepName, step); - const timeoutMs = generationTimeoutMs(spec); - yield* fs.writeFileString(path.join(result.runDir, "repair-prompt.txt"), prompt); - input.log(`REPAIR ${result.id} step=${stepName}`); - - const started = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - const agentResult = - repairProvider === "codex" - ? yield* runCodex({ - cwd: result.projectDir, - prompt, - model: result.model, - effort: result.effort, - useMcp: false, - bunx: input.bunx, - timeoutMs, - }) - : repairProvider === "agy" - ? yield* runAgy({ - cwd: result.projectDir, - prompt, - model: result.model, - effort: result.effort, - timeoutMs, - }) - : repairProvider === "pi" - ? yield* runPi({ - cwd: result.projectDir, - prompt, - model: result.model, - effort: result.effort, - timeoutMs, - }) - : repairProvider === "opencode" || repairProvider === "kilo" - ? yield* runOpencode({ - binary: repairProvider, - cwd: result.projectDir, - prompt, - model: result.model, - effort: result.effort, - useMcp: false, - bunx: input.bunx, - timeoutMs, - }) - : yield* runClaude({ - cwd: result.projectDir, - prompt, - model: result.model, - effort: result.effort, - maxBudgetUsd: input.options.maxBudgetUsd, - mcpConfig: input.emptyMcpPath, - timeoutMs, - }); - const finished = yield* Effect.clockWith((clock) => clock.currentTimeMillis); - yield* fs.writeFileString( - path.join(result.runDir, "repair.stdout.json"), - agentResult.stdout, - ); - yield* fs.writeFileString( - path.join(result.runDir, "repair.stderr.log"), - agentResult.stderr, - ); - - const parsed = - repairProvider === "codex" - ? parseCodexResult(agentResult.stdout, result.model) - : repairProvider === "agy" - ? parseAgyResult(agentResult.stdout) - : repairProvider === "pi" - ? parsePiResult(agentResult.stdout) - : repairProvider === "opencode" || repairProvider === "kilo" - ? parseOpencodeResult(agentResult.stdout) - : parseClaudeResult(agentResult.stdout); - const accounting = { - exitCode: agentResult.exitCode, - timedOut: agentResult.timedOut, - durationMs: finished - started, - resultDurationMs: parsed?.duration_ms, - outputTokens: parsed?.usage?.output_tokens, - totalCostUsd: resolvedCostUsd(repairProvider, result.model, parsed), - sessionId: parsed?.session_id, - terminalReason: parsed?.terminal_reason, - spawnError: agentResult.spawnError, - spawnErrorCode: agentResult.spawnErrorCode, - timeoutKind: agentResult.timeoutKind, - timeoutProgress: agentResult.timeoutProgress, - stderrTail: agentResult.stderrTail, - }; - const validation = yield* validateProjectCached(spec, result.projectDir, input.options); - const scored = yield* fromPromise(() => - scoreProject(spec, result.projectDir!, result.promptStyle), - ); - const repairedRun: RunResult = { - ...result, - claude: accounting, - validation, - stackScore: scored.artifact, - generatorFaithfulness: scored.faithfulness, - acceptanceScore: scored.acceptance, - outcome: undefined, - repair: undefined, - failureTags: [], - }; - repairedRun.outcome = classifyOutcome(repairedRun); - repairedRun.outcomeEvidence = outcomeEvidenceFor(repairedRun); - repairedRun.failureTags = deriveFailureTags(repairedRun); - result.repair = { - attemptedAt: new Date().toISOString(), - failingStep: stepName, - prompt, - claude: accounting, - validation, - stackScore: scored.artifact, - outcome: repairedRun.outcome, - outcomeEvidence: repairedRun.outcomeEvidence, - failureTags: repairedRun.failureTags, - } satisfies RepairResult; - yield* writeSummaryEffect( - input.options.outDir, - input.results, - input.options, - input.specs, - input.metadata, - ); - input.log(`DONE REPAIR ${result.id} outcome=${result.repair.outcome}`); - }), - { concurrency: 1, discard: true }, - ); - }); -} - -function withScaffbenchQueue( - options: ScaffbenchOptions, - log: Log, - program: Effect.Effect, -) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const lockDir = path.join(path.dirname(options.outDir), ".scaffbench.lock"); - yield* Effect.acquireRelease(acquireLock(lockDir, options.outDir, log), () => - fs.remove(lockDir, { recursive: true, force: true }).pipe(Effect.ignore), - ); - return yield* program; - }).pipe(Effect.scoped); -} - -function acquireLock(lockDir: string, outDir: string, log: Log) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - yield* fs.makeDirectory(path.dirname(lockDir), { recursive: true }); - let announcedWait = false; - - while (true) { - const acquired = yield* fs.makeDirectory(lockDir).pipe( - Effect.as(true), - Effect.catchIf( - (error) => error._tag === "SystemError" && error.reason === "AlreadyExists", - () => Effect.succeed(false), - ), - ); - if (acquired) { - yield* fs.writeFileString( - path.join(lockDir, "owner.json"), - `${JSON.stringify( - { - pid: process.pid, - outDir, - startedAt: new Date().toISOString(), - command: process.argv.join(" "), - }, - null, - 2, - )}\n`, - ); - return; - } - if (yield* removeStaleLock(lockDir)) continue; - if (!announcedWait) { - log(`QUEUE waiting for active ScaffBench run (${lockDir})`); - announcedWait = true; - } - yield* Effect.sleep(QUEUE_POLL_MS); - } - }); -} - -function removeStaleLock(lockDir: string) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const owner = yield* Effect.either( - fs - .readFileString(path.join(lockDir, "owner.json")) - .pipe(Effect.flatMap((text) => Effect.try(() => JSON.parse(text)))), - ); - if (Either.isRight(owner)) { - if (typeof owner.right.pid === "number" && isProcessAlive(owner.right.pid)) return false; - } else { - const info = yield* Effect.either(fs.stat(lockDir)); - if (Either.isLeft(info)) return true; - const modifiedAt = Option.getOrElse(info.right.mtime, () => new Date(0)).getTime(); - if (Date.now() - modifiedAt < STALE_LOCK_MS) return false; - } - yield* fs.remove(lockDir, { recursive: true, force: true }).pipe(Effect.ignore); - return true; - }); -} - -function isProcessAlive(pid: number) { - if (pid === process.pid) return true; - try { - process.kill(pid, 0); - return true; - } catch { - return false; - } -} - -function deferredValidation(projectExists: boolean, options: ScaffbenchOptions): ProjectValidation { - return projectExists - ? { - projectExists: true, - qualityGateRequested: qualityGateRequested(options), - deferred: true, - steps: {}, - } - : { - projectExists: false, - qualityGateRequested: qualityGateRequested(options), - steps: {}, - }; -} - -function needsValidation(result: RunResult, options: ScaffbenchOptions) { - if (options.skipValidation) return false; - if (!result.validation.projectExists || !result.projectDir) return false; - if (result.validation.deferred) return true; - if (options.validateExisting && options.forceRevalidate) return true; - return ( - options.validateExisting && - !result.validation.cacheKey && - Object.keys(result.validation.steps).length === 0 - ); -} - -export function validatePendingResults( - results: RunResult[], - options: ScaffbenchOptions, - specs: readonly BenchmarkSpec[], - metadata: Record, - log: Log, -) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const specsById = new Map(specs.map((spec) => [spec.id, spec])); - const pending = results - .filter((result) => needsValidation(result, options)) - .sort( - (a, b) => - validationPriority(specsById.get(a.specId)) - validationPriority(specsById.get(b.specId)), - ); - - if (pending.length === 0) { - if (options.validateExisting) log("No existing generated runs need validation."); - return; - } - - log(`VALIDATE ${pending.length} generated run${pending.length === 1 ? "" : "s"}`); - yield* Effect.forEach( - pending, - (result) => - Effect.gen(function* () { - const spec = specsById.get(result.specId); - if (!spec) return; - const projectDir = result.projectDir; - if (!projectDir || !(yield* fs.exists(projectDir))) { - result.validation = { - projectExists: false, - qualityGateRequested: qualityGateRequested(options), - steps: {}, - }; - result.outcome = classifyOutcome(result); - result.outcomeEvidence = outcomeEvidenceFor(result); - result.failureTags = deriveFailureTags(result); - yield* writeSummaryEffect(options.outDir, results, options, specs, metadata); - log(`VALIDATE ${result.id} missing archived project`); - return; - } - - log(`VALIDATE ${result.id}`); - result.validation = yield* validateProjectCached(spec, projectDir, options); - if (options.forceRevalidate) { - const scored = yield* fromPromise(() => - scoreProject(spec, projectDir, result.promptStyle), - ); - result.stackScore = scored.artifact; - result.generatorFaithfulness = scored.faithfulness; - result.acceptanceScore = scored.acceptance; - } - if (result.provenance) { - result.provenance.harnessVersion = HARNESS_VERSION; - result.provenance.validationCacheVersion = VALIDATION_CACHE_VERSION; - result.provenance.resourceProfileId = VALIDATION_RESOURCE_PROFILE_ID; - } - result.outcome = classifyOutcome(result); - result.outcomeEvidence = outcomeEvidenceFor(result); - result.failureTags = deriveFailureTags(result); - yield* writeSummaryEffect(options.outDir, results, options, specs, metadata); - log( - `DONE ${result.id} validation=${validationPassed(result)} cache=${ - result.validation.cacheHit ? "hit" : "miss" - }`, - ); - }), - { concurrency: 1, discard: true }, - ); - }); -} - -function validationPriority(spec?: BenchmarkSpec) { - if (!spec) return 50; - const native = new Set(spec.validationProfile.native ?? []); - if (native.has("cargo") || spec.family === "rust") return 100; - if (native.has("dotnet") || spec.family === "multi-ecosystem" || spec.family === "dotnet") - return 80; - return 10; -} - -function writeHarnessFiles( - outDir: string, - options: ScaffbenchOptions, - specs: readonly BenchmarkSpec[], -) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString( - path.join(outDir, "spec.json"), - `${JSON.stringify( - { - harnessVersion: HARNESS_VERSION, - selectedSpecs: specs.map((spec) => spec.id), - specs: specs.map((spec) => ({ - ...spec, - canonicalCommand: canonicalCommand(spec, ""), - })), - options: { ...options, listSpecs: undefined, writeMatrixOnly: undefined }, - }, - null, - 2, - )}\n`, - ); - }); -} - -function writeMcpConfigs(emptyMcpPath: string, bfsMcpPath: string, bunx: string) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - yield* fs.writeFileString(emptyMcpPath, `${JSON.stringify({ mcpServers: {} }, null, 2)}\n`); - yield* fs.writeFileString( - bfsMcpPath, - `${JSON.stringify( - { - mcpServers: { - "better-fullstack": { - command: bunx, - args: [bfSpec("create-better-fullstack"), "mcp"], - }, - }, - }, - null, - 2, - )}\n`, - ); - }); -} - -export function buildGenerationSchedule( - specs: readonly BenchmarkSpec[], - options: Pick, - seed: number, -) { - const schedule: Array<{ - spec: BenchmarkSpec; - effort: Effort; - pathMode: CreationPath; - trial: number; - }> = []; - let lastScheduledSpecId: string | undefined; - for (let trial = 1; trial <= options.repeats; trial += 1) { - const ordered = seededShuffle(specs, seed + trial - 1); - const firstRunnable = ordered.findIndex((spec) => resolveSpecPaths(spec, options.paths).length); - if (firstRunnable >= 0 && ordered[firstRunnable]?.id === lastScheduledSpecId) { - const swap = ordered.findIndex( - (spec, index) => - index > firstRunnable && - spec.id !== lastScheduledSpecId && - resolveSpecPaths(spec, options.paths).length > 0, - ); - if (swap >= 0) { - [ordered[firstRunnable], ordered[swap]] = [ordered[swap]!, ordered[firstRunnable]!]; - } - } - for (const spec of ordered) { - for (const effort of options.efforts) { - for (const pathMode of resolveSpecPaths(spec, options.paths)) { - schedule.push({ spec, effort, pathMode, trial }); - lastScheduledSpecId = spec.id; - } - } - } - } - return schedule; -} - -export function seededShuffle(values: readonly T[], seed: number): T[] { - const output = [...values]; - const random = mulberry32(seed >>> 0); - for (let index = output.length - 1; index > 0; index -= 1) { - const swapIndex = Math.floor(random() * (index + 1)); - [output[index], output[swapIndex]] = [output[swapIndex]!, output[index]!]; - } - return output; -} - -function mulberry32(seed: number) { - return () => { - seed |= 0; - seed = (seed + 0x6d2b79f5) | 0; - let value = Math.imul(seed ^ (seed >>> 15), 1 | seed); - value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value; - return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296; - }; -} - -export function specShuffleSeed(options: Pick) { - const digest = createHash("sha256") - .update(JSON.stringify([options.outDir, options.model, [...options.specs].sort()])) - .digest(); - return digest.readUInt32BE(0); -} - -function normalizedBudget(value: string) { - const parsed = Number.parseFloat(value); - if (!Number.isFinite(parsed) || parsed < 0) { - throw new Error(`maxBudgetUsd must be a non-negative number, got ${JSON.stringify(value)}`); - } - return parsed; -} - -export function resolvedCostUsd( - provider: ReturnType, - model: string, - parsed: - | { - usage?: { - input_tokens?: number; - output_tokens?: number; - cache_creation_input_tokens?: number; - cache_read_input_tokens?: number; - }; - total_cost_usd?: number; - } - | null - | undefined, -): number | undefined { - if (provider === "claude") return claudeCostUsd(model, parsed?.usage) ?? parsed?.total_cost_usd; - return parsed?.total_cost_usd; -} - -function qualityGateRequested(options: ScaffbenchOptions) { - return options.qualityGate; -} - -export function buildRunId( - spec: BenchmarkSpec, - model: string, - effort: Effort, - pathMode: CreationPath, - trial: number, - _repeats: number, -) { - const base = `${spec.id}-${model}-${effort}-${pathMode}`; - return `${base}-r${String(trial).padStart(2, "0")}`; -} - -function legacyRunId(spec: BenchmarkSpec, model: string, effort: Effort, pathMode: CreationPath) { - return `${spec.id}-${model}-${effort}-${pathMode}`; -} - -export function findCompletedTrial( - results: readonly RunResult[], - spec: BenchmarkSpec, - model: string, - effort: Effort, - pathMode: CreationPath, - trial: number, -) { - const currentId = buildRunId(spec, model, effort, pathMode, trial, 1); - const oldId = legacyRunId(spec, model, effort, pathMode); - return results.find((result) => result.id === currentId || (trial === 1 && result.id === oldId)); -} - -export function assertResumeProtocol(input: { - recorded?: RunProtocol; - current: RunProtocol; - results: readonly RunResult[]; - schedule: ReturnType; - model: string; -}) { - if (input.recorded === undefined) return; - if (input.recorded.repeats !== input.current.repeats) { - throw new Error( - `runProtocol conflict: this out-dir was launched with repeats=${input.recorded.repeats} and ` + - `cannot resume as repeats=${input.current.repeats}; start a fresh out-dir for a different repeat count`, - ); - } - if (input.recorded.seed === input.current.seed) return; - - const aligned = input.results.every((result) => { - const scheduled = input.schedule.find( - (entry) => - entry.spec.id === result.specId && - entry.effort === result.effort && - entry.pathMode === result.path && - entry.trial === (result.trial ?? 1), - ); - if (!scheduled || result.model !== input.model) return false; - const expected = buildRunId( - scheduled.spec, - input.model, - scheduled.effort, - scheduled.pathMode, - scheduled.trial, - input.current.repeats, - ); - return ( - result.id === expected || - (scheduled.trial === 1 && - result.id === - legacyRunId(scheduled.spec, input.model, scheduled.effort, scheduled.pathMode)) - ); - }); - if (!aligned) { - throw new Error( - `runProtocol conflict: recorded repeats=${String(input.recorded?.repeats)} seed=${String( - input.recorded?.seed, - )}; current repeats=${input.current.repeats} seed=${input.current.seed}; existing artifacts do not align`, - ); - } -} - -function buildProjectName( - spec: BenchmarkSpec, - pathMode: CreationPath, - effort: Effort, - trial: number, - repeats: number, -) { - const base = `sb21-${spec.id}-${pathMode}-${effort}`.replace(/[^a-zA-Z0-9_-]/g, "-"); - return repeats === 1 ? base : `${base}-r${String(trial).padStart(2, "0")}`; -} - -function readExistingSummary(outDir: string) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const summaryPath = path.join(outDir, "summary.json"); - if (!(yield* fs.exists(summaryPath))) return undefined; - const text = yield* fs.readFileString(summaryPath); - return yield* Effect.try({ - try: () => JSON.parse(text) as unknown, - catch: (cause) => - new Error( - `${summaryPath} is unreadable (${cause instanceof Error ? cause.message : String(cause)}); ` + - "repair or remove it before resuming this out-dir", - ), - }); - }); -} - -function asRecord(value: unknown): Record | undefined { - return typeof value === "object" && value !== null && !Array.isArray(value) - ? (value as Record) - : undefined; -} - -function stringList(value: unknown): string[] | undefined { - return Array.isArray(value) && value.every((item) => typeof item === "string") - ? [...(value as string[])] - : undefined; -} - -function recordedRunResults(summary: unknown): RunResult[] { - const results = asRecord(summary)?.results; - return Array.isArray(results) ? (results as RunResult[]) : []; -} - -function completedResults(summary: unknown): RunResult[] { - return recordedRunResults(summary).filter(isCompletedHarnessRun); -} - -export function recordedRunOptions(summary: unknown): Partial { - const options = asRecord(asRecord(summary)?.options); - if (!options) return {}; - const recorded: Partial = {}; - if (typeof options.model === "string") recorded.model = options.model; - if (Number.isInteger(options.repeats)) recorded.repeats = options.repeats as number; - if (typeof options.maxBudgetUsd === "string") recorded.maxBudgetUsd = options.maxBudgetUsd; - if (options.promptStyle === "explicit" || options.promptStyle === "natural") { - recorded.promptStyle = options.promptStyle; - } - const efforts = stringList(options.efforts)?.filter((effort): effort is Effort => - EFFORT_VALUES.includes(effort as Effort), - ); - if (efforts && efforts.length > 0) recorded.efforts = efforts; - const paths = stringList(options.paths)?.filter((pathMode): pathMode is CreationPath => - CREATION_PATH_VALUES.includes(pathMode as CreationPath), - ); - if (paths && paths.length > 0) recorded.paths = paths; - const specs = stringList(options.specs); - if (specs && specs.length > 0) recorded.specs = specs; - return recorded; -} - -export function topUpSpecSelection(requested: readonly string[], recorded: readonly string[]) { - if (recorded.every((id) => requested.includes(id))) return [...recorded]; - const unknown = requested.filter((id) => !recorded.includes(id)); - if (unknown.length > 0) { - throw new Error( - `--top-up: ${unknown.join(", ")} not in this out-dir's recorded specs (${recorded.join(", ")})`, - ); - } - return [...requested]; -} - -function topUpRecords(value: unknown): TopUpRecord[] | undefined { - if (!Array.isArray(value)) return undefined; - const records = value.flatMap((item) => { - const record = asRecord(item); - const specs = stringList(record?.specs); - return Number.isInteger(record?.trials) && specs && typeof record?.recordedAt === "string" - ? [{ trials: record.trials as number, specs, recordedAt: record.recordedAt }] - : []; - }); - return records.length > 0 ? records : undefined; -} - -export function recordedRunProtocol(summary: unknown): RunProtocol | undefined { - const record = asRecord(summary); - const metadata = asRecord(record?.metadata); - const protocol = asRecord(metadata?.runProtocol); - if (Number.isInteger(protocol?.repeats) && Number.isInteger(protocol?.seed)) { - const topUps = topUpRecords(protocol!.topUps); - return { - repeats: protocol!.repeats as number, - seed: protocol!.seed as number, - ...(topUps ? { topUps } : {}), - }; - } - const options = asRecord(record?.options); - if (Number.isInteger(options?.repeats) && Number.isInteger(metadata?.specOrderSeed)) { - return { repeats: options!.repeats as number, seed: metadata!.specOrderSeed as number }; - } - return undefined; -} - -function readExistingResults(outDir: string) { - return readExistingSummary(outDir).pipe(Effect.map(completedResults)); -} - -export function isCompletedHarnessRun(result: RunResult) { - const outcome = classifyOutcome(result); - return outcome !== "provider-infra" && outcome !== "harness-infra"; -} - -export function assertScheduleSupported( - schedule: ReturnType, - provider: ReturnType, - model: string, -) { - if ((provider === "agy" || provider === "pi") && schedule.some((e) => e.pathMode === "mcp")) { - throw new Error( - `--paths mcp is not supported by the ${provider} adapter (${model}): it wires no MCP server, ` + - "so the agent would receive a prompt requiring tools it cannot call", - ); - } - if (provider === "agy") { - for (const effort of new Set(schedule.map((entry) => entry.effort))) { - agyModelString(model, effort); - } - } -} - -function writeSummaryEffect( - outDir: string, - results: readonly RunResult[], - options: ScaffbenchOptions, - specs: readonly BenchmarkSpec[], - metadata: Record, -) { - return fromPromise(() => writeSummary(outDir, results, options, specs, metadata)); -} diff --git a/scripts/scaffbench/scoring.ts b/scripts/scaffbench/scoring.ts deleted file mode 100644 index 6fbd0f3ca..000000000 --- a/scripts/scaffbench/scoring.ts +++ /dev/null @@ -1,656 +0,0 @@ -import type { - BenchmarkSpec, - CommandDisciplineCheck, - CommandResult, - CreationPath, - FailureTag, - ProjectIndex, - PromptStyle, - OutcomeEvidence, - RunOutcome, - RunOutcomeRollup, - RunResult, - ScaffbenchOptions, - StackScore, - StepResult, - ToolCompliance, -} from "@scaffbench/types"; - -import { ESTIMATED_BUDGET_TOLERANCE, SCAFFBENCH_SPEC_SCORE_WEIGHTS } from "@scaffbench/constants"; -import { isRecurringTransientFailure } from "@scaffbench/validation/classification"; -import { walk, parseJsonc } from "@scaffbench/validation/shared"; -import { existsSync } from "node:fs"; -import { readFile, readdir, stat } from "node:fs/promises"; -import path from "node:path"; - -export function typecheckGate( - scripts: Record, - hasTsconfig: boolean, -): "check-types" | "typecheck" | "tsc" | null { - if (scripts["check-types"]) return "check-types"; - if (scripts.typecheck) return "typecheck"; - if (hasTsconfig) return "tsc"; - return null; -} - -export async function scoreArtifact( - spec: BenchmarkSpec, - projectDir: string, - promptStyle: PromptStyle = "explicit", -): Promise { - return scoreMarkers(spec, await collectProjectIndex(projectDir), promptStyle); -} - -function scoredMarkers(spec: BenchmarkSpec, promptStyle: PromptStyle) { - return promptStyle === "natural" - ? spec.strictMarkers.filter((marker) => !marker.explicitOnly) - : spec.strictMarkers; -} - -export async function scoreProject( - spec: BenchmarkSpec, - projectDir: string, - promptStyle: PromptStyle = "explicit", -): Promise<{ artifact: StackScore; faithfulness?: StackScore; acceptance?: StackScore }> { - const index = await collectProjectIndex(projectDir); - const artifact = scoreMarkers(spec, index, promptStyle); - const btsPath = path.join(projectDir, "bts.jsonc"); - const faithfulness = existsSync(btsPath) - ? scoreBts(spec, await readFile(btsPath, "utf8")) - : undefined; - const acceptance = - promptStyle === "natural" && spec.acceptanceSets - ? scoreAcceptance(spec.acceptanceSets, index) - : undefined; - return { artifact, faithfulness, acceptance }; -} - -function scoreAcceptance( - acceptanceSets: Record, - index: ProjectIndex, -): StackScore { - const deps = [...index.dependencies]; - const files = [...index.files]; - const capabilities = Object.entries(acceptanceSets); - const misses: string[] = []; - let matched = 0; - for (const [capability, accepted] of capabilities) { - const satisfied = accepted.some((pattern) => acceptancePatternMatch(pattern, deps, files)); - if (satisfied) matched += 1; - else misses.push(capability); - } - return scoreFromCounts(matched, capabilities.length, misses); -} - -function acceptancePatternMatch( - pattern: string, - deps: readonly string[], - files: readonly string[], -): boolean { - if (pattern.startsWith(".")) { - return files.some((file) => file === pattern || file.includes(`${pattern}/`)); - } - const prefix = pattern.endsWith("/") ? pattern : `${pattern}/`; - return deps.some((dep) => dep === pattern || dep.startsWith(prefix)); -} - -export function scoreBts(spec: BenchmarkSpec, raw: string): StackScore { - const config = parseJsonc(raw); - if (!config) return emptyScore(spec); - - if (spec.expectedParts?.length) { - return scoreStackParts(spec, config); - } - - const misses: string[] = []; - let matched = 0; - let total = 0; - - for (const [key, expected] of Object.entries(spec.expectedConfig ?? {})) { - const expectedValues = Array.isArray(expected) ? expected : [expected]; - const actual = config[key]; - total += expectedValues.length; - if (Array.isArray(actual)) { - for (const expectedValue of expectedValues) { - if (actual.includes(expectedValue)) matched += 1; - else misses.push(`${key}: missing ${expectedValue}`); - } - } else { - const expectedValue = expectedValues[0]; - if (actual === expectedValue) matched += 1; - else misses.push(`${key}: expected ${expectedValue}, got ${String(actual)}`); - } - } - - for (const addon of spec.expectedAddons ?? []) { - total += 1; - if (Array.isArray(config.addons) && config.addons.includes(addon)) matched += 1; - else misses.push(`addons: missing ${addon}`); - } - - return scoreFromCounts(matched, total, misses); -} - -function scoreStackParts(spec: BenchmarkSpec, config: Record): StackScore { - const actualParts = new Set(formatConfigStackParts(config.stackParts ?? [])); - const misses: string[] = []; - let matched = 0; - let total = 0; - - for (const expectedPart of spec.expectedParts ?? []) { - total += 1; - if (actualParts.has(expectedPart)) matched += 1; - else misses.push(`stackParts: missing ${expectedPart}`); - } - - for (const addon of spec.expectedAddons ?? []) { - total += 1; - if (Array.isArray(config.addons) && config.addons.includes(addon)) matched += 1; - else misses.push(`addons: missing ${addon}`); - } - - return scoreFromCounts(matched, total, misses); -} - -function formatConfigStackParts(stackParts: readonly Record[]) { - const byId = new Map(stackParts.map((part) => [part.id, part])); - return stackParts - .filter((part) => part.source !== "provided") - .map((part) => { - if (!part.ownerPartId) return `${part.role}:${part.ecosystem}:${part.toolId}`; - const owner = byId.get(part.ownerPartId); - const ownerRole = owner?.role ?? part.ownerPartId.split(":")[0] ?? "backend"; - return `${ownerRole}.${part.role}:${part.ecosystem}:${part.toolId}`; - }); -} - -function scoreMarkers( - spec: BenchmarkSpec, - index: ProjectIndex, - promptStyle: PromptStyle = "explicit", -): StackScore { - const misses: string[] = []; - const markers = scoredMarkers(spec, promptStyle); - let matched = 0; - - for (const marker of markers) { - const depsMatch = - !marker.deps || marker.deps.every((dep) => depMarkerMatches(index.dependencies, dep)); - const sourceMatch = - !marker.source || marker.source.every((pattern) => index.sourceText.includes(pattern)); - const textMatch = - !marker.text || marker.text.every((pattern) => index.allText.includes(pattern)); - const textAnyMatch = - !marker.textAny || marker.textAny.some((pattern) => index.allText.includes(pattern)); - const filesMatch = - !marker.files || marker.files.every((pattern) => fileMarkerMatches(index.files, pattern)); - const forbiddenDepsMatch = - !marker.forbiddenDeps || - marker.forbiddenDeps.every((dep) => !depMarkerMatches(index.dependencies, dep)); - const forbiddenTextMatch = - !marker.forbiddenText || - marker.forbiddenText.every((pattern) => !index.allText.includes(pattern)); - const forbiddenFilesMatch = - !marker.forbiddenFiles || - marker.forbiddenFiles.every((pattern) => !fileMarkerMatches(index.files, pattern)); - - if ( - depsMatch && - sourceMatch && - textMatch && - textAnyMatch && - filesMatch && - forbiddenDepsMatch && - forbiddenTextMatch && - forbiddenFilesMatch - ) { - matched += 1; - } else { - misses.push(marker.id); - } - } - - return scoreFromCounts(matched, markers.length, misses); -} - -export function depMarkerMatches(dependencies: ReadonlySet, pattern: string) { - if (!pattern.endsWith("/")) return dependencies.has(pattern); - for (const dep of dependencies) { - if (dep.startsWith(pattern)) return true; - } - return false; -} - -export function fileMarkerMatches(files: ReadonlySet, pattern: string) { - if (files.has(pattern)) return true; - const matcher = fileMarkerPattern(pattern); - for (const file of files) { - if (matcher.test(file.split(path.sep).join("/"))) return true; - } - return false; -} - -function fileMarkerPattern(pattern: string) { - const source = pattern - .split("/") - .map((segment) => - segment - .split("*") - .map((literal) => literal.replace(/[|\\{}()[\]^$+*?.-]/g, "\\$&")) - .join("[^/]*"), - ) - .join("/"); - return new RegExp(`(^|/)${source}$`); -} - -function scoreFromCounts(matched: number, total: number, misses: string[]): StackScore { - return { - matched, - total, - percent: total > 0 ? Math.round((matched / total) * 100) : 0, - misses, - }; -} - -export function emptyArtifactScore( - spec: BenchmarkSpec, - promptStyle: PromptStyle = "explicit", -): StackScore { - return { - matched: 0, - total: scoredMarkers(spec, promptStyle).length, - percent: 0, - misses: ["project not found or unscorable"], - }; -} - -export function emptyAcceptanceScore(spec: BenchmarkSpec): StackScore { - return { - matched: 0, - total: Object.keys(spec.acceptanceSets ?? {}).length, - percent: 0, - misses: ["project not found"], - }; -} - -function emptyScore(spec: BenchmarkSpec): StackScore { - const total = - (spec.expectedParts?.length ?? 0) + - Object.values(spec.expectedConfig ?? {}).reduce( - (sum, value) => sum + (Array.isArray(value) ? value.length : 1), - 0, - ) + - (spec.expectedAddons?.length ?? 0); - return { - matched: 0, - total: total || spec.strictMarkers.length, - percent: 0, - misses: ["project not found or unscorable"], - }; -} - -async function collectProjectIndex(projectDir: string): Promise { - const index: ProjectIndex = { - dependencies: new Set(), - files: new Set(), - packageText: "", - sourceText: "", - configText: "", - allText: "", - }; - - await walk(projectDir, async (filePath) => { - const relativePath = path.relative(projectDir, filePath); - index.files.add(relativePath); - if (path.basename(filePath) === "bts.jsonc") return; - if ( - !/(package\.json|Cargo\.toml|go\.mod|pyproject\.toml|pom\.xml|mix\.exs|\.csproj|\.gradle|\.kts|\.ts|\.tsx|\.js|\.jsx|\.mjs|\.cjs|\.rs|\.go|\.py|\.cs|\.java|\.kt|\.exs|\.ex|\.heex|\.html|\.vue|\.svelte|\.json|\.jsonc|\.proto|\.toml|\.yml|\.yaml)$/.test( - filePath, - ) - ) { - return; - } - const info = await stat(filePath); - if (info.size > 250_000) return; - const content = await readFile(filePath, "utf8"); - index.allText += `\n${content}`; - - if (path.basename(filePath) === "package.json") { - index.packageText += `\n${content}`; - collectPackageDependencies(index.dependencies, content); - return; - } - - if (/\.(ts|tsx|js|jsx|mjs|cjs|rs|go|py|cs)$/.test(filePath)) { - index.sourceText += `\n${content}`; - return; - } - - index.configText += `\n${content}`; - }); - - return index; -} - -function collectPackageDependencies(target: Set, rawPackageJson: string) { - try { - const parsed = JSON.parse(rawPackageJson); - for (const section of [ - "dependencies", - "devDependencies", - "peerDependencies", - "optionalDependencies", - ]) { - for (const dep of Object.keys(parsed[section] ?? {})) { - target.add(dep); - } - } - } catch {} -} - -export function extractToolUses(stdout: string): { name: string; command?: string }[] { - const uses: { name: string; command?: string }[] = []; - for (const line of stdout.split("\n")) { - const trimmed = line.trim(); - if (!trimmed.startsWith("{")) continue; - let event: any; - try { - event = JSON.parse(trimmed); - } catch { - continue; - } - const content = event?.message?.content; - if (Array.isArray(content)) { - for (const block of content) { - if (block?.type === "tool_use" && typeof block.name === "string") { - const command = - typeof block.input?.command === "string" ? block.input.command : undefined; - uses.push({ name: block.name, command }); - } - } - } - if (event?.type === "item.completed" && event.item) { - const item = event.item; - if (item.type === "mcp_tool_call" && typeof item.tool === "string") { - uses.push({ name: item.tool }); - } else if (item.type === "command_execution" && typeof item.command === "string") { - uses.push({ name: "bash", command: item.command }); - } - } - if (event?.part?.type === "tool" && typeof event.part.tool === "string") { - const command = - typeof event.part.state?.input?.command === "string" - ? event.part.state.input.command - : undefined; - uses.push({ name: event.part.tool, command }); - } - if (event?.type === "tool_execution_start" && typeof event.toolName === "string") { - const command = typeof event.args?.command === "string" ? event.args.command : undefined; - uses.push({ name: event.toolName, command }); - } - } - return uses; -} - -export async function scoreToolCompliance( - pathMode: CreationPath, - projectDir: string | null, - claude: CommandResult, -): Promise { - const toolUses = extractToolUses(claude.stdout); - const hasBtsConfig = projectDir ? existsSync(path.join(projectDir, "bts.jsonc")) : false; - - const usedBfsCreate = toolUses.some((use) => /bfs_create_project/i.test(use.name)); - const usedAnyBfsTool = toolUses.some((use) => /bfs_/i.test(use.name)); - const bashCommands = toolUses - .filter((use) => /(^|_)bash$/i.test(use.name)) - .map((use) => (use.command ?? "").toLowerCase()); - const isBfsCli = (cmd: string) => /create\s+better-fullstack|create-better-fullstack/.test(cmd); - const ranBfsCli = bashCommands.some(isBfsCli); - - const checks: CommandDisciplineCheck[] = []; - if (pathMode === "prompt") { - checks.push({ - id: "no-bf-config", - status: hasBtsConfig ? "fail" : "pass", - detail: "prompt-only must not produce bts.jsonc", - }); - checks.push({ - id: "no-bf-tool", - status: usedAnyBfsTool || ranBfsCli ? "fail" : "pass", - detail: "prompt-only must not call a Better-Fullstack MCP tool or CLI", - }); - } else { - checks.push({ - id: "used-mcp", - status: usedBfsCreate || hasBtsConfig ? "pass" : "fail", - detail: "MCP path must call bfs_create_project", - }); - checks.push({ - id: "no-cli-create", - status: ranBfsCli ? "fail" : "pass", - detail: "MCP path must not run bun create better-fullstack", - }); - } - - const score = checks.filter((check) => check.status === "pass").length; - return { score, total: checks.length, checks }; -} - -const ADVISORY_STEP_KEYS = new Set(["lint", "format", "test", "doctor", "route", "tidy"]); - -export function stepBaseName(name: string) { - return name.slice(name.lastIndexOf(":") + 1); -} -export function isAdvisoryStep(name: string) { - return ADVISORY_STEP_KEYS.has(stepBaseName(name)); -} - -function applicableSteps(result: RunResult, predicate: (name: string) => boolean): StepResult[] { - return Object.entries(result.validation.steps) - .filter((entry): entry is [string, StepResult] => Boolean(entry[1])) - .filter(([name, step]) => step.status !== "na" && predicate(name)) - .map(([, step]) => step); -} - -function stepsAllGreen(steps: readonly StepResult[]) { - return steps.every( - (step) => step.status !== "skip" && step.exitCode === 0 && !step.timedOut && !step.spawnError, - ); -} - -export function validationPassed(result: RunResult) { - if (result.validation.deferred) return false; - if (!result.validation.projectExists) return false; - const core = applicableSteps(result, (name) => !isAdvisoryStep(name)); - if (core.length === 0) return false; - return stepsAllGreen(core); -} - -const QUALITY_TIER_STEPS = new Set(["lint", "format"]); - -function isQualityTierStep(name: string) { - return QUALITY_TIER_STEPS.has(stepBaseName(name)); -} - -export function specScore(result: RunResult) { - const core = validationPassed(result) ? 1 : 0; - const gates = applicableSteps(result, isQualityTierStep); - const quality = - core === 0 || gates.length === 0 - ? 0 - : gates.filter((gate) => stepsAllGreen([gate])).length / gates.length; - const stack = result.stackScore.percent / 100; - const weights = SCAFFBENCH_SPEC_SCORE_WEIGHTS; - return { - core, - quality, - stack, - score: weights.core * core + weights.quality * quality + weights.stack * stack, - }; -} - -export function qualityPassed(result: RunResult) { - if (result.validation.skipped || result.validation.qualityGateRequested !== true) { - return "na" as const; - } - if (!validationPassed(result)) return false; - return stepsAllGreen(applicableSteps(result, isQualityTierStep)); -} - -const BUDGET_TERMINAL_REASON = /budget|cost[_-]?limit|max[_-]?cost|spend/i; - -export function isBudgetExhausted(result: RunResult) { - if (result.claude.terminalReason && BUDGET_TERMINAL_REASON.test(result.claude.terminalReason)) { - return true; - } - const policy = result.budgetPolicy; - return Boolean( - policy && - !policy.budgetEnforced && - typeof result.claude.totalCostUsd === "number" && - result.claude.totalCostUsd > policy.maxBudgetUsd * ESTIMATED_BUDGET_TOLERANCE, - ); -} - -export function outcomeEvidenceFor(result: RunResult): OutcomeEvidence | undefined { - if (result.validation.skipped) return undefined; - const policy = result.budgetPolicy; - if ( - policy && - !policy.budgetEnforced && - !BUDGET_TERMINAL_REASON.test(result.claude.terminalReason ?? "") && - typeof result.claude.totalCostUsd === "number" && - result.claude.totalCostUsd > policy.maxBudgetUsd * ESTIMATED_BUDGET_TOLERANCE - ) { - return { budgetEstimated: true }; - } - return undefined; -} - -export function classifyOutcome(result: RunResult): RunOutcome { - if (result.validation.skipped) return "skipped"; - if (isBudgetExhausted(result)) return "budget-exhausted"; - if (result.claude.timedOut) return "deadline-exhausted"; - if (result.claude.spawnError) return "harness-infra"; - if (result.validation.deferred) return "validation-infra"; - if (!validationPassed(result) && hasProviderInfraEvidence(result)) return "provider-infra"; - - const coreEntries = Object.entries(result.validation.steps).filter( - ([name, step]) => step && step.status !== "na" && !isAdvisoryStep(name), - ) as [string, StepResult][]; - if (coreEntries.some(([name, step]) => isRecurringTransientFailure(name, step))) { - return "validation-infra"; - } - if (hasSoleHarnessBlocker(coreEntries)) return "harness-infra"; - return validationPassed(result) ? "success" : "model-failure"; -} - -export function rollupOutcome(outcome: RunOutcome): RunOutcomeRollup { - if (outcome === "success") return "success"; - if (["provider-infra", "harness-infra", "validation-infra", "skipped"].includes(outcome)) { - return "infra-inconclusive"; - } - return "model-failure"; -} - -export function scoredOutcome(result: RunResult) { - if (classifyOutcome(result) === "skipped") return false; - return rollupOutcome(classifyOutcome(result)) !== "infra-inconclusive"; -} - -function hasProviderInfraEvidence(result: RunResult) { - const reason = `${result.claude.terminalReason ?? ""}\n${result.claude.stderrTail ?? ""}`; - if (/(?:opencode-unknown|pi)-zero-usage-no-tools/.test(reason) && !result.claude.outputTokens) { - return true; - } - if (/opencode-unknown-zero-usage-step/.test(reason)) return true; - const providerWithError = - /\b(?:provider|upstream|endpoint|gateway)\b[^\n]{0,80}\b(?:error|fail(?:ed|ure)?|unavailable|timeout|timed out|rejected|denied)\b|\b(?:error|fail(?:ed|ure)?|unavailable|timeout|timed out|rejected|denied)\b[^\n]{0,80}\b(?:provider|upstream|endpoint|gateway)\b/i; - const explicitInfraError = - /\b(?:HTTP(?:\/\d(?:\.\d)?)?\s*429|status(?:\s+code)?\D{0,10}429|too many requests|rate.?limit(?:ed)?[\s:_-]+(?:error|exceeded|reached|hit)|unauthori[sz]ed|ECONNRESET|ETIMEDOUT)\b/i; - const contextualCapacity = - /\b(?:overloaded|capacity|authentication)\b[^\n]{0,40}\b(?:error|fail(?:ed|ure)?|unavailable|exhausted|exceeded|rejected|denied)\b|\b(?:error|fail(?:ed|ure)?|unavailable|exhausted|exceeded|rejected|denied)\b[^\n]{0,40}\b(?:overloaded|capacity|authentication)\b/i; - return ( - providerWithError.test(reason) || - explicitInfraError.test(reason) || - contextualCapacity.test(reason) - ); -} - -function hasSoleHarnessBlocker(core: readonly [string, StepResult][]) { - const blockers = core.filter(([, step]) => step.spawnError || isDotnetSdkNotFound(step)); - if (blockers.length === 0) return false; - return !core.some( - ([, step]) => - !step.spawnError && - !isDotnetSdkNotFound(step) && - step.status !== "skip" && - step.exitCode !== null && - step.exitCode !== 0, - ); -} - -const DOTNET_SDK_NOT_FOUND = - /A compatible \.NET SDK was not found|Install the \[[^\]]*\] \.NET SDK/; -function isDotnetSdkNotFound(step: StepResult) { - if (step.exitCode === 0 || step.exitCode === null) return false; - return ( - DOTNET_SDK_NOT_FOUND.test(step.stderrTail ?? "") || - DOTNET_SDK_NOT_FOUND.test(step.stdoutTail ?? "") - ); -} - -export function deriveFailureTags(result: RunResult): FailureTag[] { - const tags = new Set(); - if (result.validation.deferred) tags.add("validation-deferred"); - if (result.claude.timedOut) tags.add("claude-timeout"); - if (result.claude.exitCode !== 0) tags.add("claude-error"); - const outcome = classifyOutcome(result); - if (outcome === "budget-exhausted") tags.add("budget-exhausted"); - if (outcome === "deadline-exhausted") tags.add("deadline-exhausted"); - if (outcome === "provider-infra") tags.add("provider-infra"); - if (outcome === "harness-infra") tags.add("harness-infra"); - if (outcome === "validation-infra") tags.add("validation-infra"); - if (result.claude.timeoutProgress) tags.add(result.claude.timeoutProgress); - if (!result.validation.projectExists) tags.add("project-not-found"); - if (result.stackScore.matched < result.stackScore.total) tags.add("stack-mismatch"); - if ( - result.generatorFaithfulness && - result.generatorFaithfulness.percent === 100 && - result.stackScore.percent < 100 - ) { - tags.add("stack-unwired"); - } - if (result.toolCompliance.checks.some((check) => check.status === "fail")) { - tags.add("tool-violation"); - tags.add("command-discipline"); - } - - for (const [name, step] of Object.entries(result.validation.steps)) { - if (!step || (step.exitCode === 0 && !step.timedOut)) continue; - if (step.spawnError || isDotnetSdkNotFound(step)) { - tags.add("toolchain-missing"); - continue; - } - if (isRecurringTransientFailure(name, step)) continue; - const base = stepBaseName(name).toLowerCase(); - if (base.includes("install") || base.includes("restore")) tags.add("install-failed"); - if (base.includes("build") || base.includes("cargocheck")) tags.add("build-failed"); - if (base.includes("typecheck")) tags.add("typecheck-failed"); - if (base.includes("lint")) tags.add("lint-failed"); - if (base.includes("format")) tags.add("format-failed"); - if (base.includes("test")) tags.add("test-failed"); - if (base.includes("doctor")) tags.add("doctor-failed"); - if (base.includes("route")) tags.add("route-failed"); - } - - if ( - !result.validation.deferred && - !validationPassed(result) && - rollupOutcome(outcome) === "model-failure" - ) { - tags.add("validation-failed"); - } - return [...tags].sort(); -} diff --git a/scripts/scaffbench/specs/ai-search-workbench.ts b/scripts/scaffbench/specs/ai-search-workbench.ts deleted file mode 100644 index e95488ad4..000000000 --- a/scripts/scaffbench/specs/ai-search-workbench.ts +++ /dev/null @@ -1,124 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -import { AI_SEARCH_ADDONS, AI_SEARCH_FLAGS, AI_SEARCH_STACK } from "@scaffbench/constants"; - -export const AiSearchWorkbenchSpec: BenchmarkSpec = { - id: "ai-search-workbench", - introducedAt: "2026-08-21", - title: "AI search workbench with split semantic/full-text search on the Vite+ toolchain", - lane: "core", - difficulty: 2, - family: "typescript", - supportedByBetterFullstack: true, - requirements: [ - "Create a TypeScript monorepo for an AI support/search workbench: agents triage tickets, search past resolutions semantically, and admins search raw documents.", - "Use a TanStack Router web app styled with Tailwind and shadcn/ui.", - "Use a Hono backend on Bun.", - "Use oRPC for the app API.", - "Use PostgreSQL with Drizzle for relational app data only.", - "Use Better Auth for accounts.", - "Use the Vercel AI SDK for AI features.", - "Use Qdrant for semantic vector search over embeddings. Postgres is present, but pgvector is not acceptable here, the vector workload must live in Qdrant.", - "Use OpenSearch for full-text document/admin search. Do not collapse the two search modes into one engine.", - "Use Inngest for background indexing jobs.", - "Use Pino logging and OpenTelemetry instrumentation.", - "Use TanStack Store, TanStack Form, Valibot, Vitest + Playwright, and Paraglide.", - "Use the Vite+ toolchain (`vp`) for workspace scripts, linting, and formatting. Do not add Turborepo, Nx, or Biome.", - "Include DevContainer and GitHub Actions CI output.", - "Do not add payments, email, realtime, CMS, file upload, file storage, feature flags, or deploy targets.", - ], - naturalPrompt: - "Build a production-grade AI support search starter. Support agents need semantic search over embedded past resolutions, admins need full-text search over raw documents, and the two workloads should not share one engine. It also needs account auth, relational app data, background indexing, observability, tests, i18n, CI, and a workspace toolchain for tasks, lint, and format. Keep it deploy-target neutral and do not include commerce/email/storage extras.", - rightLibraryNotes: [ - "Qdrant must be used for semantic vector search; pgvector is not acceptable even though Postgres is present.", - "OpenSearch must be used for full-text document/admin search.", - "Inngest must be used for background indexing jobs.", - "oRPC must be used for the app API.", - "The Vite+ toolchain (vp) replaces Turborepo/Nx and Biome for workspace tasks, lint, and format.", - ], - canonicalFlags: AI_SEARCH_FLAGS, - expectedConfig: AI_SEARCH_STACK, - expectedAddons: AI_SEARCH_ADDONS, - strictMarkers: [ - { id: "frontend:tanstack-router", deps: ["@tanstack/react-router"] }, - { id: "css:tailwind", deps: ["tailwindcss"] }, - { id: "ui:shadcn", files: ["components.json"] }, - { id: "backend:hono", deps: ["hono"] }, - { id: "api:orpc", deps: ["@orpc/server"], source: ["@orpc/"] }, - { id: "database:postgres+drizzle", deps: ["drizzle-orm"], source: ["drizzle-orm"] }, - { id: "auth:better-auth", deps: ["better-auth"], source: ["better-auth"] }, - { id: "ai:vercel-ai", deps: ["ai"] }, - { - id: "vectorDb:qdrant", - deps: ["@qdrant/js-client-rest"], - source: ["@qdrant/js-client-rest"], - }, - { - id: "search:opensearch", - deps: ["@opensearch-project/opensearch"], - source: ["@opensearch-project/opensearch"], - }, - { id: "jobQueue:inngest", deps: ["inngest"], source: ["inngest"] }, - { id: "logging:pino", deps: ["pino"], source: ["pino"] }, - { id: "observability:opentelemetry", deps: ["@opentelemetry/api"] }, - { id: "state:tanstack-store", deps: ["@tanstack/store"] }, - { id: "forms:tanstack-form", deps: ["@tanstack/react-form"] }, - { id: "validation:valibot", deps: ["valibot"] }, - { id: "testing:vitest-playwright", deps: ["vitest", "@playwright/test"] }, - { - id: "i18n:paraglide", - deps: ["@inlang/paraglide-js"], - files: ["project.inlang/settings.json"], - }, - { id: "addon:vite-plus", deps: ["vite-plus"] }, - { id: "addon:devcontainer", files: [".devcontainer/devcontainer.json"] }, - { id: "addon:github-actions", files: [".github/workflows/*.y*ml"] }, - { id: "forbidden:pgvector", forbiddenDeps: ["pgvector"] }, - { id: "forbidden:turborepo", forbiddenDeps: ["turbo"] }, - { id: "forbidden:nx", forbiddenDeps: ["nx", "@nx/"], forbiddenFiles: ["nx.json"] }, - { id: "forbidden:biome", forbiddenDeps: ["@biomejs/biome"] }, - { id: "forbidden:payments", forbiddenDeps: ["stripe", "@stripe/stripe-js", "polar-sh"] }, - { id: "forbidden:email", forbiddenDeps: ["resend", "nodemailer", "@react-email/components"] }, - ], - acceptanceSets: { - "web-framework": ["@tanstack/react-router", "next", "react-router", "@remix-run", "vite"], - backend: ["hono", "express", "fastify", "elysia", "@nestjs/core"], - "relational-db": ["drizzle-orm", "@prisma/client", "kysely", "typeorm", "sequelize"], - auth: ["better-auth", "lucia", "@clerk", "next-auth", "@auth/", "@workos-inc"], - ai: ["ai", "@ai-sdk", "openai", "@anthropic-ai", "langchain"], - "semantic-search": [ - "@qdrant/js-client-rest", - "pgvector", - "weaviate-ts-client", - "@pinecone-database/pinecone", - "chromadb", - "@zilliz/milvus2-sdk-node", - ], - "full-text-search": [ - "@opensearch-project/opensearch", - "@elastic/elasticsearch", - "meilisearch", - "typesense", - "algoliasearch", - ], - "background-jobs": [ - "inngest", - "bullmq", - "@trigger.dev", - "graphile-worker", - "pg-boss", - "bee-queue", - ], - observability: ["@opentelemetry/api", "pino", "winston", "@sentry", "@logtail"], - testing: ["vitest", "jest", "@playwright/test", "cypress"], - i18n: ["@inlang/paraglide-js", "next-intl", "i18next", "react-i18next", "lingui"], - toolchain: ["vite-plus", "turbo", "nx"], - ci: [".github/workflows", ".gitlab-ci.yml", ".circleci"], - }, - validationProfile: { - packageManager: "bun", - qualityGate: true, - doctorCheck: true, - routeCheckCandidate: true, - }, -}; diff --git a/scripts/scaffbench/specs/dotnet-blazor-cqrs.ts b/scripts/scaffbench/specs/dotnet-blazor-cqrs.ts deleted file mode 100644 index b319aec51..000000000 --- a/scripts/scaffbench/specs/dotnet-blazor-cqrs.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const DotnetBlazorCqrsSpec: BenchmarkSpec = { - id: "dotnet-blazor-cqrs", - introducedAt: "2026-08-21", - title: ".NET Blazor app with Dapper, Duende IdentityServer, and HotChocolate GraphQL", - lane: "core", - difficulty: 2, - family: "dotnet", - supportedByBetterFullstack: true, - requirements: [ - "Create a .NET project using ASP.NET Blazor (not Minimal API or MVC).", - "Use Dapper for data access (not EF Core).", - "Use Duende IdentityServer for auth (not ASP.NET Identity).", - "Use HotChocolate for a GraphQL API (not Minimal API or gRPC).", - "Use PostgreSQL.", - "Use NUnit with Moq and Testcontainers for .NET (not xUnit).", - "Use Quartz.NET for background jobs (not Hangfire).", - "Use SignalR for realtime.", - "Use OpenTelemetry, NLog, and health checks for observability (not Serilog).", - "Use FluentValidation, Redis caching, and Docker deploy output.", - ], - naturalPrompt: - "Build a .NET starter for an internal operations console. It needs a C# web UI, lightweight data access, a dedicated identity server, a GraphQL API, Postgres, background scheduling, realtime updates, validation, caching, observability, and container output. Choose the right .NET libraries rather than the framework defaults.", - rightLibraryNotes: [ - "Blazor is required for the web framework.", - "Dapper is required for data access (not EF Core).", - "Duende IdentityServer is required for auth (not ASP.NET Identity).", - "HotChocolate GraphQL is required (not Minimal API), with NUnit and Quartz.NET.", - ], - canonicalFlags: [ - "--ecosystem", - "dotnet", - "--database", - "postgres", - "--dotnet-web-framework", - "aspnet-blazor", - "--dotnet-orm", - "dapper", - "--dotnet-auth", - "duende-identityserver", - "--dotnet-api", - "graphql-hotchocolate", - "--dotnet-testing", - "nunit", - "moq", - "testcontainers-dotnet", - "--dotnet-job-queue", - "quartz-net", - "--dotnet-realtime", - "signalr", - "--dotnet-observability", - "opentelemetry-dotnet", - "nlog", - "health-checks", - "--dotnet-validation", - "fluentvalidation", - "--dotnet-caching", - "redis", - "--dotnet-deploy", - "docker", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "claude-md", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "dotnet", - database: "postgres", - dotnetWebFramework: "aspnet-blazor", - dotnetOrm: "dapper", - dotnetAuth: "duende-identityserver", - dotnetApi: "graphql-hotchocolate", - dotnetTesting: ["nunit", "moq", "testcontainers-dotnet"], - dotnetJobQueue: "quartz-net", - dotnetRealtime: "signalr", - dotnetObservability: ["opentelemetry-dotnet", "nlog", "health-checks"], - dotnetValidation: "fluentvalidation", - dotnetCaching: "redis", - dotnetDeploy: "docker", - }, - strictMarkers: [ - { id: "dotnet:blazor", text: ["RazorComponents"] }, - { id: "orm:dapper", text: ["Dapper"] }, - { id: "auth:duende", text: ["Duende"] }, - { id: "api:hotchocolate", text: ["HotChocolate"] }, - { id: "testing:nunit", text: ["NUnit"] }, - { id: "testing:moq", text: ["Moq"] }, - { id: "testing:testcontainers", text: ["Testcontainers"] }, - { id: "jobs:quartz", text: ["Quartz"] }, - { id: "realtime:signalr", text: ["SignalR"] }, - { id: "validation:fluentvalidation", text: ["FluentValidation"] }, - { id: "observability:nlog", text: ["NLog"] }, - { id: "observability:opentelemetry", textAny: ["OpenTelemetry"] }, - { - id: "observability:health-checks", - textAny: ["AddHealthChecks", "MapHealthChecks", "HealthChecks"], - }, - { id: "db:postgres", textAny: ["Npgsql"] }, - { - id: "caching:redis", - textAny: ["StackExchange.Redis", "AddStackExchangeRedisCache", "Redis"], - }, - { id: "deploy:docker", files: ["Dockerfile"] }, - { id: "forbidden:hangfire", forbiddenText: ["Hangfire"] }, - { id: "forbidden:serilog", forbiddenText: ["Serilog"] }, - { id: "forbidden:ef-core", forbiddenText: ["Microsoft.EntityFrameworkCore"] }, - { id: "forbidden:aspnet-identity", forbiddenText: ["Microsoft.AspNetCore.Identity"] }, - { id: "forbidden:grpc", forbiddenText: ["Grpc.AspNetCore", "Grpc.Net.Client"] }, - { id: "forbidden:xunit", forbiddenText: ["xunit"] }, - { - id: "forbidden:mvc", - forbiddenText: ["AddControllersWithViews", "MapControllerRoute", "AddMvc("], - }, - { id: "forbidden:minimal-api-host", forbiddenText: ["CreateSlimBuilder"] }, - ], - validationProfile: { native: ["dotnet"] }, -}; diff --git a/scripts/scaffbench/specs/elixir-broadway-absinthe.ts b/scripts/scaffbench/specs/elixir-broadway-absinthe.ts deleted file mode 100644 index f1ea398ae..000000000 --- a/scripts/scaffbench/specs/elixir-broadway-absinthe.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const ElixirBroadwayAbsintheSpec: BenchmarkSpec = { - id: "elixir-broadway-absinthe", - introducedAt: "2026-08-21", - title: "Elixir Phoenix LiveView app with Absinthe, Broadway, Oban, and Nx", - lane: "core", - difficulty: 2, - family: "elixir", - supportedByBetterFullstack: true, - requirements: [ - "Create an Elixir project using Phoenix LiveView (not plain Phoenix).", - "Use Ecto SQL with PostgreSQL and Ecto changesets for validation.", - "Use Guardian for auth. Do not generate or hand-write a phx.gen.auth user store (no bcrypt password hashing, no UserSessionController) and do not use Ueberauth.", - "Use Absinthe for a GraphQL API.", - "Include Broadway and Nx as libraries.", - "Use Phoenix Presence for realtime, Oban for jobs, Finch as the HTTP client, Jason for JSON, Swoosh for email, Nebulex for caching, and PromEx for observability.", - "Use Wallaby for testing, Dialyxir for code quality (not Credo), and Fly for deploy output.", - ], - naturalPrompt: - "Build an Elixir Phoenix starter for a realtime data-ingestion app. It needs server-rendered live views, Postgres via Ecto, a dedicated JWT auth library, a GraphQL API, data pipelines, numerical/ML support, presence tracking, durable background jobs, a pooled HTTP client, caching, Prometheus metrics, browser-based tests, static analysis, and a deploy target. Pick the right BEAM libraries rather than the framework defaults.", - rightLibraryNotes: [ - "Phoenix LiveView is required (not plain Phoenix).", - "Guardian is required for auth; Absinthe is required for the GraphQL API.", - "Broadway and Oban are required for pipelines and jobs.", - "Presence, Finch, Nebulex, PromEx, Wallaby, and Dialyxir are the required choices.", - ], - canonicalFlags: [ - "--ecosystem", - "elixir", - "--database", - "postgres", - "--elixir-web-framework", - "phoenix-live-view", - "--elixir-orm", - "ecto-sql", - "--elixir-auth", - "guardian", - "--elixir-api", - "absinthe", - "--elixir-libraries", - "broadway", - "nx", - "--elixir-realtime", - "presence", - "--elixir-jobs", - "oban", - "--elixir-validation", - "ecto-changesets", - "--elixir-http", - "finch", - "--elixir-json", - "jason", - "--elixir-email", - "swoosh", - "--elixir-caching", - "nebulex", - "--elixir-observability", - "prom_ex", - "--elixir-testing", - "wallaby", - "--elixir-quality", - "dialyxir", - "--elixir-deploy", - "fly", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "claude-md", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "elixir", - database: "postgres", - elixirWebFramework: "phoenix-live-view", - elixirOrm: "ecto-sql", - elixirAuth: "guardian", - elixirApi: "absinthe", - elixirLibraries: ["broadway", "nx"], - elixirRealtime: "presence", - elixirJobs: "oban", - elixirValidation: "ecto-changesets", - elixirHttp: "finch", - elixirJson: "jason", - elixirEmail: "swoosh", - elixirCaching: "nebulex", - elixirObservability: "prom_ex", - elixirTesting: "wallaby", - elixirQuality: "dialyxir", - elixirDeploy: "fly", - }, - strictMarkers: [ - { id: "web:phoenix-live-view", text: ["phoenix_live_view"] }, - { id: "orm:ecto-sql", text: ["ecto_sql"] }, - { id: "auth:guardian", text: [":guardian"] }, - { id: "api:absinthe", text: ["absinthe"] }, - { id: "lib:broadway", text: ["broadway"] }, - { id: "lib:nx", text: ["{:nx,"] }, - { id: "jobs:oban", text: [":oban"] }, - { id: "http:finch", text: [":finch"] }, - { id: "caching:nebulex", text: ["nebulex"] }, - { id: "observability:prom_ex", text: ["prom_ex"] }, - { id: "testing:wallaby", text: ["wallaby"] }, - { id: "quality:dialyxir", text: ["dialyxir"] }, - { id: "realtime:presence", textAny: ["Presence"] }, - { id: "validation:ecto-changesets", textAny: ["changeset"] }, - { id: "json:jason", textAny: [":jason", "Jason"] }, - { id: "email:swoosh", textAny: [":swoosh", "Swoosh"] }, - { id: "deploy:fly", files: ["fly.toml"] }, - { id: "forbidden:credo", forbiddenText: [":credo"] }, - { id: "forbidden:ueberauth", forbiddenText: ["ueberauth"] }, - { - id: "forbidden:phx-gen-auth", - forbiddenText: [":bcrypt_elixir", "UserSessionController", "phx.gen.auth"], - }, - ], - validationProfile: { native: ["elixir"] }, -}; diff --git a/scripts/scaffbench/specs/frontier-effect-eventsourcing.ts b/scripts/scaffbench/specs/frontier-effect-eventsourcing.ts deleted file mode 100644 index 1dccd3f96..000000000 --- a/scripts/scaffbench/specs/frontier-effect-eventsourcing.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const FrontierEffectEventsourcingSpec: BenchmarkSpec = { - id: "frontier-effect-eventsourcing", - introducedAt: "2026-08-21", - title: - "Frontier: TypeScript Effect service with event-sourcing/CQRS and tRPC-over-WebSocket subscriptions", - lane: "core", - difficulty: 3, - family: "typescript", - supportedByBetterFullstack: false, - paths: ["prompt"], - requirements: [ - "Create a TypeScript backend for a bank-ledger service built on the Effect ecosystem (effect runtime, services, layers).", - "Implement event-sourcing with CQRS: an append-only event store, write-side command handlers (open account, deposit, withdraw with overdraft rejection), and read-side balance projections.", - "Projections must be rebuildable by replaying the event store from zero, and applying an event twice must not corrupt a projection.", - "Expose the API via tRPC, including a subscription over WebSockets that streams balance updates from the read model.", - "Include an outbox pattern for reliable event publication.", - "Provide build and type-check scripts.", - ], - naturalPrompt: - "Build a TypeScript bank-ledger backend on the Effect ecosystem that uses event sourcing with CQRS, an append-only event store, command handlers for open/deposit/withdraw with overdraft rejection on the write side, replayable idempotent balance projections on the read side, and an outbox for reliable publishing. Expose it through tRPC, including a WebSocket subscription that streams balance updates.", - rightLibraryNotes: [ - "The service layer must be built on Effect.", - "Use event-sourcing + CQRS (event store, projections, outbox), not plain CRUD.", - "Projections must be replayable and idempotent.", - "Expose tRPC with a WebSocket subscription for the read model.", - ], - canonicalFlags: [], - strictMarkers: [ - { id: "runtime:effect", deps: ["effect"] }, - { id: "api:trpc", deps: ["@trpc/server"] }, - { id: "ws:subscription", text: ["subscription"] }, - { id: "pattern:event-sourcing", text: ["projection"] }, - { id: "store:event-store", textAny: ["eventStore", "EventStore", "event_store"] }, - { id: "store:append-only", textAny: ["append"] }, - { id: "cqrs:commands", textAny: ["command", "Command"] }, - { id: "command:open-account", textAny: ["openAccount", "OpenAccount", "open_account"] }, - { id: "command:deposit", textAny: ["deposit", "Deposit"] }, - { id: "command:withdraw", textAny: ["withdraw", "Withdraw"] }, - { - id: "ledger:overdraft", - textAny: ["overdraft", "Overdraft", "insufficient", "Insufficient"], - }, - { id: "projection:replay", textAny: ["replay", "Replay", "rebuild", "Rebuild"] }, - { id: "projection:idempotent", textAny: ["idempot", "Idempot"] }, - { id: "pattern:outbox", textAny: ["outbox", "Outbox"] }, - ], - validationProfile: { packageManager: "bun" }, -}; diff --git a/scripts/scaffbench/specs/frontier-polyglot-proto.ts b/scripts/scaffbench/specs/frontier-polyglot-proto.ts deleted file mode 100644 index 75d2b6816..000000000 --- a/scripts/scaffbench/specs/frontier-polyglot-proto.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const FrontierPolyglotProtoSpec: BenchmarkSpec = { - id: "frontier-polyglot-proto", - introducedAt: "2026-08-21", - title: - "Frontier: polyglot monorepo, shared protobuf across a Rust gRPC service, a Go gateway, and a TS client", - lane: "core", - difficulty: 3, - family: "multi-ecosystem", - supportedByBetterFullstack: false, - paths: ["prompt"], - requirements: [ - "Create one monorepo with a single shared Protocol Buffers (proto3) service contract.", - "Implement the core service in Rust using Tonic for gRPC.", - "Implement an edge gateway in Go that speaks gRPC to the Rust service and exposes HTTP/JSON.", - "Implement a TypeScript web client generated from the same proto contract.", - "Wire codegen so all three consume the one .proto definition; provide build scripts per package.", - ], - naturalPrompt: - "Build a polyglot monorepo around a single service contract: a Rust gRPC core service, a Go gateway that bridges gRPC to HTTP/JSON, and a TypeScript client, all generated from one shared Protocol Buffers definition. Set up the codegen and per-package builds so the three stay in sync.", - rightLibraryNotes: [ - "A single shared proto3 contract must drive all three languages.", - "Rust uses Tonic for the gRPC service; Go uses grpc-go for the gateway.", - "The TypeScript client must be generated from the same proto.", - ], - canonicalFlags: [], - strictMarkers: [ - { id: "proto:proto3", text: ["proto3"] }, - { id: "proto:contract-file", files: ["*.proto"] }, - { id: "rust:tonic", text: ["tonic"] }, - { id: "go:grpc", text: ["google.golang.org/grpc"] }, - { id: "ts:protobuf", text: ["protobuf"] }, - { id: "codegen:from-proto", textAny: ["buf.gen", "protoc", "tonic_build", "protoc-gen"] }, - { id: "rust:proto-generated", textAny: ["prost", "tonic_build", "include_proto"] }, - { - id: "go:proto-generated", - textAny: ["google.golang.org/protobuf", "protoc-gen-go", ".pb.go"], - }, - { - id: "ts:proto-generated", - textAny: [ - "ts-proto", - "@bufbuild/protobuf", - "@connectrpc/connect", - "google-protobuf", - "protobufjs", - "_pb.ts", - "_pb.js", - ], - }, - { id: "gateway:http-json", textAny: ["net/http", "grpc-gateway", "gin-gonic", "go-chi"] }, - ], - prerequisiteCommands: [ - { - command: ["buf", "generate"], - whenConfigFound: ["buf.gen.yaml", "buf.gen.yml", "buf.gen.json"], - }, - ], - validationProfile: { packageManager: "bun", native: ["cargo", "go"] }, -}; diff --git a/scripts/scaffbench/specs/go-realtime-api.ts b/scripts/scaffbench/specs/go-realtime-api.ts deleted file mode 100644 index 6d1c504e0..000000000 --- a/scripts/scaffbench/specs/go-realtime-api.ts +++ /dev/null @@ -1,107 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const GoRealtimeApiSpec: BenchmarkSpec = { - id: "go-realtime-api", - introducedAt: "2026-08-21", - title: "Go realtime API with Chi, Ent, gRPC, NATS, Redis, and OpenTelemetry", - lane: "core", - difficulty: 1, - family: "go", - supportedByBetterFullstack: true, - requirements: [ - "Create a Go API project for a fleet-tracking admin service: vehicles report positions, operators watch them live.", - "Use Chi as the router, not Gin/Echo/Fiber.", - "Use PostgreSQL with Ent.", - "Use gRPC for the vehicle-ingest service contract.", - "Use Cobra CLI tooling, Zap logging, JWT auth, Testify + GoMock tests, Gorilla WebSocket for live position updates, NATS messaging, Redis caching, Viper config, and OpenTelemetry.", - ], - naturalPrompt: - "Build a Go backend starter for a realtime admin API. It needs a lightweight router, Ent/Postgres models, typed gRPC service contracts, CLI/admin commands, structured logging, auth, websocket updates, event messaging, Redis cache, configuration, tracing, and test doubles.", - rightLibraryNotes: [ - "Chi is required as the web framework.", - "Ent is required for the data layer.", - "gRPC-Go is required for typed service contracts.", - "NATS and Gorilla WebSocket are required for messaging and realtime updates.", - ], - canonicalFlags: [ - "--ecosystem", - "go", - "--database", - "postgres", - "--go-web-framework", - "chi", - "--go-orm", - "ent", - "--go-api", - "grpc-go", - "--go-cli", - "cobra", - "--go-logging", - "zap", - "--go-auth", - "jwt", - "--go-testing", - "testify", - "gomock", - "--go-realtime", - "gorilla-websocket", - "--go-message-queue", - "nats", - "--go-caching", - "redis", - "--go-config", - "viper", - "--go-observability", - "opentelemetry", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "none", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "go", - database: "postgres", - goWebFramework: "chi", - goOrm: "ent", - goApi: "grpc-go", - goCli: "cobra", - goLogging: "zap", - goAuth: "jwt", - goTesting: ["testify", "gomock"], - goRealtime: "gorilla-websocket", - goMessageQueue: "nats", - goCaching: "redis", - goConfig: "viper", - goObservability: "opentelemetry", - }, - strictMarkers: [ - { id: "backend:chi", text: ["github.com/go-chi/chi"] }, - { id: "orm:ent", text: ["entgo.io/ent"], files: ["ent/schema/*.go"] }, - { id: "db:postgres", textAny: ["github.com/lib/pq", "github.com/jackc/pgx"] }, - { id: "api:grpc-go", text: ["google.golang.org/grpc"] }, - { id: "cli:cobra", text: ["github.com/spf13/cobra"] }, - { id: "logging:zap", text: ["go.uber.org/zap"] }, - { id: "auth:jwt", text: ["github.com/golang-jwt/jwt"] }, - { id: "testing:testify+gomock", text: ["github.com/stretchr/testify", "go.uber.org/mock"] }, - { id: "realtime:gorilla-websocket", text: ["github.com/gorilla/websocket"] }, - { id: "queue:nats", text: ["github.com/nats-io/nats.go"] }, - { id: "caching:redis", text: ["github.com/redis/go-redis"] }, - { id: "config:viper", text: ["github.com/spf13/viper"] }, - { id: "observability:opentelemetry", text: ["go.opentelemetry.io/otel"] }, - { id: "forbidden:gin", forbiddenText: ["github.com/gin-gonic/gin"] }, - { id: "forbidden:echo", forbiddenText: ["github.com/labstack/echo"] }, - { id: "forbidden:fiber", forbiddenText: ["github.com/gofiber/fiber"] }, - ], - validationProfile: { native: ["go"] }, -}; diff --git a/scripts/scaffbench/specs/index.ts b/scripts/scaffbench/specs/index.ts deleted file mode 100644 index 13f4cc52f..000000000 --- a/scripts/scaffbench/specs/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { BenchmarkSpec, SpecDifficulty } from "@scaffbench/types"; - -import { AiSearchWorkbenchSpec } from "@scaffbench/specs/ai-search-workbench"; -import { DotnetBlazorCqrsSpec } from "@scaffbench/specs/dotnet-blazor-cqrs"; -import { ElixirBroadwayAbsintheSpec } from "@scaffbench/specs/elixir-broadway-absinthe"; -import { FrontierEffectEventsourcingSpec } from "@scaffbench/specs/frontier-effect-eventsourcing"; -import { FrontierPolyglotProtoSpec } from "@scaffbench/specs/frontier-polyglot-proto"; -import { GoRealtimeApiSpec } from "@scaffbench/specs/go-realtime-api"; -import { JavaSpringJooqKeycloakSpec } from "@scaffbench/specs/java-spring-jooq-keycloak"; -import { MultiDotnetOpsSpec } from "@scaffbench/specs/multi-dotnet-ops"; -import { MultiTsGoGrpcSpec } from "@scaffbench/specs/multi-ts-go-grpc"; -import { PythonIngestionApiSpec } from "@scaffbench/specs/python-ingestion-api"; -import { ReactNativeExpoSpec } from "@scaffbench/specs/react-native-expo"; -import { RustLeptosAxumSpec } from "@scaffbench/specs/rust-leptos-axum"; -import { TsMinimalRestraintSpec } from "@scaffbench/specs/ts-minimal-restraint"; -import { TsSvelteEdgeOrpcSpec } from "@scaffbench/specs/ts-svelte-edge-orpc"; - -export const SCAFFBENCH_2_SPECS: readonly BenchmarkSpec[] = [ - AiSearchWorkbenchSpec, - RustLeptosAxumSpec, - PythonIngestionApiSpec, - GoRealtimeApiSpec, - MultiDotnetOpsSpec, - TsMinimalRestraintSpec, - TsSvelteEdgeOrpcSpec, - DotnetBlazorCqrsSpec, - MultiTsGoGrpcSpec, - JavaSpringJooqKeycloakSpec, - ElixirBroadwayAbsintheSpec, - ReactNativeExpoSpec, - FrontierPolyglotProtoSpec, - FrontierEffectEventsourcingSpec, -]; - -export function specDifficulty(specId: string): SpecDifficulty { - return SCAFFBENCH_2_SPECS.find((spec) => spec.id === specId)?.difficulty ?? 1; -} diff --git a/scripts/scaffbench/specs/java-spring-jooq-keycloak.ts b/scripts/scaffbench/specs/java-spring-jooq-keycloak.ts deleted file mode 100644 index 77d42c447..000000000 --- a/scripts/scaffbench/specs/java-spring-jooq-keycloak.ts +++ /dev/null @@ -1,152 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const JavaSpringJooqKeycloakSpec: BenchmarkSpec = { - id: "java-spring-jooq-keycloak", - introducedAt: "2026-08-21", - title: "Java Spring Boot API with jOOQ, Keycloak, GraphQL, and property/architecture tests", - lane: "core", - difficulty: 1, - family: "java", - supportedByBetterFullstack: true, - requirements: [ - "Create a Java project using Spring Boot with the Maven build tool.", - "Use jOOQ for data access (NOT Spring Data JPA).", - "Use Keycloak as the identity provider. Do not hand-roll authentication and do not use Spring Security as the identity provider (no in-app user store, form login, or `UserDetailsService`); wiring Keycloak through Spring Security's OAuth2 resource server is the expected integration.", - "Use Spring for GraphQL for the API and Logback for logging.", - "Use PostgreSQL.", - "Include MapStruct, Resilience4j, Spring for Kafka, Spring Batch, Micrometer Prometheus, Caffeine, springdoc-openapi, OpenTelemetry, Spring Validation, and Spring Actuator.", - "Include JUnit 5, Mockito, Testcontainers, AssertJ, REST Assured, WireMock, Awaitility, ArchUnit, and jqwik for testing.", - ], - naturalPrompt: - "Build a Java Spring Boot starter for an event-driven service. It needs Postgres data access with a type-safe SQL layer, a dedicated identity server for auth, a GraphQL API, fault tolerance, event streaming, batch jobs, metrics, mapping, API docs, and tracing, plus a serious test stack with mocks, containers, HTTP stubs, architecture rules, and property-based tests. Choose the right Java libraries rather than the Spring defaults.", - rightLibraryNotes: [ - "jOOQ is required for data access; Spring Data JPA is not used.", - "Keycloak is required as the identity provider; an in-app Spring Security user store or hand-rolled auth is a failure, while Spring Security's OAuth2 resource server is the expected way to validate Keycloak tokens.", - "Spring for GraphQL is required for the API.", - "ArchUnit and jqwik are required (architecture + property-based testing).", - ], - canonicalFlags: [ - "--ecosystem", - "java", - "--database", - "postgres", - "--java-web-framework", - "spring-boot", - "--java-build-tool", - "maven", - "--java-orm", - "jooq", - "--java-auth", - "keycloak", - "--java-api", - "spring-graphql", - "--java-logging", - "logback", - "--java-libraries", - "mapstruct", - "resilience4j", - "spring-kafka", - "spring-batch", - "micrometer-prometheus", - "caffeine", - "springdoc-openapi", - "opentelemetry-java", - "spring-validation", - "spring-actuator", - "--java-testing-libraries", - "junit5", - "mockito", - "testcontainers", - "assertj", - "rest-assured", - "wiremock", - "awaitility", - "archunit", - "jqwik", - "--auth", - "none", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "claude-md", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "java", - database: "postgres", - javaWebFramework: "spring-boot", - javaBuildTool: "maven", - javaOrm: "jooq", - javaAuth: "keycloak", - javaApi: "spring-graphql", - javaLogging: "logback", - javaLibraries: [ - "mapstruct", - "resilience4j", - "spring-kafka", - "spring-batch", - "micrometer-prometheus", - "caffeine", - "springdoc-openapi", - "opentelemetry-java", - "spring-validation", - "spring-actuator", - ], - javaTestingLibraries: [ - "junit5", - "mockito", - "testcontainers", - "assertj", - "rest-assured", - "wiremock", - "awaitility", - "archunit", - "jqwik", - ], - }, - strictMarkers: [ - { id: "backend:spring-boot", text: ["spring-boot-starter-parent"] }, - { id: "build:maven", files: ["pom.xml"] }, - { id: "orm:jooq", text: ["jooq"] }, - { id: "auth:keycloak", textAny: ["keycloak", "Keycloak", "KEYCLOAK"] }, - { - id: "auth:oauth2-resource-server", - textAny: ["oauth2-resource-server", "oauth2ResourceServer", "issuer-uri", "issuerUri"], - }, - { id: "api:spring-graphql", text: ["spring-boot-starter-graphql"] }, - { id: "logging:logback", textAny: ["logback"] }, - { id: "lib:mapstruct", text: ["mapstruct"] }, - { id: "lib:resilience4j", text: ["resilience4j"] }, - { id: "lib:spring-kafka", text: ["spring-kafka"] }, - { id: "lib:spring-batch", text: ["spring-boot-starter-batch"] }, - { id: "lib:micrometer-prometheus", text: ["micrometer-registry-prometheus"] }, - { id: "lib:caffeine", textAny: ["caffeine"] }, - { id: "lib:springdoc-openapi", textAny: ["springdoc"] }, - { id: "lib:opentelemetry", textAny: ["opentelemetry"] }, - { id: "lib:spring-validation", textAny: ["spring-boot-starter-validation"] }, - { id: "lib:spring-actuator", textAny: ["spring-boot-starter-actuator"] }, - { id: "testing:junit5", textAny: ["junit-jupiter", "spring-boot-starter-test"] }, - { id: "testing:mockito", textAny: ["mockito"] }, - { id: "testing:assertj", textAny: ["assertj"] }, - { id: "testing:rest-assured", textAny: ["rest-assured", "restassured"] }, - { id: "testing:wiremock", textAny: ["wiremock", "WireMock"] }, - { id: "testing:awaitility", textAny: ["awaitility"] }, - { id: "testing:archunit", text: ["archunit"] }, - { id: "testing:jqwik", text: ["jqwik"] }, - { id: "testing:testcontainers", text: ["testcontainers"] }, - { id: "forbidden:jpa", forbiddenText: ["spring-boot-starter-data-jpa"] }, - { - id: "forbidden:in-app-auth", - forbiddenText: ["UserDetailsService", "formLogin", "InMemoryUserDetailsManager"], - }, - ], - validationProfile: { native: ["java"] }, -}; diff --git a/scripts/scaffbench/specs/multi-dotnet-ops.ts b/scripts/scaffbench/specs/multi-dotnet-ops.ts deleted file mode 100644 index e3e809aea..000000000 --- a/scripts/scaffbench/specs/multi-dotnet-ops.ts +++ /dev/null @@ -1,134 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const MultiDotnetOpsSpec: BenchmarkSpec = { - id: "multi-dotnet-ops", - introducedAt: "2026-08-21", - title: "Multi-ecosystem ops portal with TypeScript frontend and .NET Minimal API backend", - lane: "core", - difficulty: 2, - family: "multi-ecosystem", - supportedByBetterFullstack: true, - requirements: [ - "Create one multi-ecosystem project graph for an incident-ops portal: on-call engineers acknowledge incidents in the web UI, the backend fans out notifications.", - "Use a Next.js TypeScript frontend with Tailwind and shadcn/ui.", - "Use an ASP.NET Minimal API backend.", - "Use EF Core, ASP.NET Identity, Minimal API endpoints, xUnit, Testcontainers for .NET, Serilog, SignalR for live incident updates, FluentValidation, Hangfire for notification fan-out, memory cache, and Docker output.", - "Use PostgreSQL as the shared database.", - "Include Turborepo, Biome, and GitHub Actions.", - ], - naturalPrompt: - "Build a multi-ecosystem ops portal starter: a TypeScript web frontend and a .NET backend. It needs Postgres-backed identity, API endpoints, validation, background jobs, realtime notifications, observability/logging, tests, containers, and CI. Use the project graph instead of forcing everything into one ecosystem.", - rightLibraryNotes: [ - "The frontend must be TypeScript Next.js.", - "The backend must be ASP.NET Minimal API.", - "EF Core and ASP.NET Identity are required.", - "Hangfire and SignalR are required for jobs and realtime updates.", - ], - canonicalFlags: [ - "--part", - "frontend:typescript:next", - "--part", - "frontend.css:typescript:tailwind", - "--part", - "frontend.ui:typescript:shadcn-ui", - "--part", - "backend:dotnet:aspnet-minimal", - "--part", - "backend.orm:dotnet:ef-core", - "--part", - "backend.auth:dotnet:aspnet-identity", - "--part", - "backend.api:dotnet:minimal-api", - "--part", - "backend.testing:dotnet:xunit", - "--part", - "backend.testing:dotnet:testcontainers-dotnet", - "--part", - "backend.observability:dotnet:serilog", - "--part", - "backend.realtime:dotnet:signalr", - "--part", - "backend.validation:dotnet:fluentvalidation", - "--part", - "backend.jobQueue:dotnet:hangfire", - "--part", - "backend.caching:dotnet:memory-cache", - "--part", - "backend.deploy:dotnet:docker", - "--part", - "database:universal:postgres", - "--addons", - "turborepo", - "biome", - "github-actions", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--shadcn-base", - "radix", - "--shadcn-style", - "nova", - "--shadcn-icon-library", - "lucide", - "--shadcn-color-theme", - "neutral", - "--shadcn-base-color", - "neutral", - "--shadcn-font", - "inter", - "--shadcn-radius", - "default", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedParts: [ - "frontend:typescript:next", - "frontend.css:typescript:tailwind", - "frontend.ui:typescript:shadcn-ui", - "backend:dotnet:aspnet-minimal", - "backend.orm:dotnet:ef-core", - "backend.auth:dotnet:aspnet-identity", - "backend.api:dotnet:minimal-api", - "backend.testing:dotnet:xunit", - "backend.testing:dotnet:testcontainers-dotnet", - "backend.observability:dotnet:serilog", - "backend.realtime:dotnet:signalr", - "backend.validation:dotnet:fluentvalidation", - "backend.jobQueue:dotnet:hangfire", - "backend.caching:dotnet:memory-cache", - "backend.deploy:dotnet:docker", - "database:universal:postgres", - ], - expectedAddons: ["turborepo", "biome", "github-actions"], - strictMarkers: [ - { id: "frontend:next", deps: ["next"] }, - { id: "frontend:tailwind", deps: ["tailwindcss"] }, - { id: "frontend:shadcn", files: ["components.json"] }, - { id: "backend:aspnet-minimal", files: ["Program.cs"], text: ["MapGet"] }, - { id: "orm:ef-core", text: ["Microsoft.EntityFrameworkCore"] }, - { id: "db:postgres", textAny: ["Npgsql"] }, - { id: "auth:aspnet-identity", text: ["Microsoft.AspNetCore.Identity"] }, - { id: "testing:xunit", text: ["xunit"] }, - { id: "testing:testcontainers", text: ["Testcontainers"] }, - { id: "logging:serilog", text: ["Serilog"] }, - { id: "realtime:signalr", text: ["SignalR"] }, - { id: "validation:fluentvalidation", text: ["FluentValidation"] }, - { id: "jobs:hangfire", text: ["Hangfire"] }, - { - id: "caching:memory-cache", - textAny: ["IMemoryCache", "AddMemoryCache", "Microsoft.Extensions.Caching.Memory"], - }, - { id: "deploy:docker", files: ["Dockerfile"] }, - { id: "addon:turborepo", deps: ["turbo"] }, - { id: "addon:biome", deps: ["@biomejs/biome"] }, - { id: "addon:github-actions", files: [".github/workflows/*.y*ml"] }, - ], - validationProfile: { - packageManager: "bun", - native: ["dotnet"], - qualityGate: true, - doctorCheck: true, - }, -}; diff --git a/scripts/scaffbench/specs/multi-ts-go-grpc.ts b/scripts/scaffbench/specs/multi-ts-go-grpc.ts deleted file mode 100644 index 5ca940336..000000000 --- a/scripts/scaffbench/specs/multi-ts-go-grpc.ts +++ /dev/null @@ -1,116 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const MultiTsGoGrpcSpec: BenchmarkSpec = { - id: "multi-ts-go-grpc", - introducedAt: "2026-08-21", - title: "Multi-ecosystem app: Nuxt (Vue) frontend with a Go Chi + gRPC backend", - lane: "core", - difficulty: 2, - family: "multi-ecosystem", - supportedByBetterFullstack: true, - requirements: [ - "Create one multi-ecosystem project graph for a live auction dashboard: a Vue web frontend over a Go bid-processing backend.", - "Use a Nuxt (Vue) TypeScript frontend with Tailwind.", - "Use a Go backend with the Chi router (not Gin/Echo/Fiber).", - "Use sqlc for data access (not GORM or Ent).", - "Use gRPC-Go for typed service contracts.", - "Use goth for auth, Centrifuge for realtime bid updates, Watermill for messaging, Ristretto for caching, koanf for config, and zerolog for logging.", - "Use OpenTelemetry and Testify + GoMock.", - "Use PostgreSQL as the shared database.", - ], - naturalPrompt: - "Build a multi-ecosystem starter for a live auction dashboard: a Vue/Nuxt web frontend and a Go backend. The Go side needs a lightweight router, type-safe SQL, typed gRPC contracts, social auth, scalable realtime, a messaging abstraction, an in-process cache, config management, structured logging, tracing, and test doubles. Use the project graph instead of one ecosystem.", - rightLibraryNotes: [ - "The frontend must be TypeScript Nuxt (Vue).", - "The Go backend must use Chi, sqlc, and gRPC-Go.", - "Centrifuge and Watermill are required for realtime and messaging.", - "Ristretto, koanf, and zerolog are required (not Redis, Viper, zap).", - ], - canonicalFlags: [ - "--part", - "frontend:typescript:nuxt", - "--part", - "frontend.css:typescript:tailwind", - "--part", - "backend:go:chi", - "--part", - "backend.orm:go:sqlc", - "--part", - "backend.api:go:grpc-go", - "--part", - "backend.auth:go:goth", - "--part", - "backend.logging:go:zerolog", - "--part", - "backend.realtime:go:centrifuge", - "--part", - "backend.jobQueue:go:watermill", - "--part", - "backend.caching:go:ristretto", - "--part", - "backend.config:go:koanf", - "--part", - "backend.observability:go:opentelemetry", - "--part", - "backend.testing:go:testify", - "--part", - "backend.testing:go:gomock", - "--part", - "database:universal:postgres", - "--addons", - "turborepo", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedParts: [ - "frontend:typescript:nuxt", - "frontend.css:typescript:tailwind", - "backend:go:chi", - "backend.orm:go:sqlc", - "backend.api:go:grpc-go", - "backend.auth:go:goth", - "backend.logging:go:zerolog", - "backend.realtime:go:centrifuge", - "backend.jobQueue:go:watermill", - "backend.caching:go:ristretto", - "backend.config:go:koanf", - "backend.observability:go:opentelemetry", - "backend.testing:go:testify", - "backend.testing:go:gomock", - "database:universal:postgres", - ], - expectedAddons: ["turborepo"], - strictMarkers: [ - { id: "frontend:nuxt", deps: ["nuxt"] }, - { id: "frontend:tailwind", deps: ["tailwindcss"] }, - { id: "backend:chi", text: ["github.com/go-chi/chi"] }, - { id: "orm:sqlc", files: ["sqlc.*"] }, - { id: "db:postgres", textAny: ["lib/pq", "jackc/pgx", "postgres"] }, - { id: "api:grpc-go", text: ["google.golang.org/grpc"] }, - { id: "auth:goth", text: ["github.com/markbates/goth"] }, - { id: "realtime:centrifuge", text: ["github.com/centrifugal/centrifuge"] }, - { id: "queue:watermill", text: ["github.com/ThreeDotsLabs/watermill"] }, - { id: "caching:ristretto", text: ["github.com/dgraph-io/ristretto"] }, - { id: "config:koanf", text: ["github.com/knadh/koanf"] }, - { id: "logging:zerolog", text: ["github.com/rs/zerolog"] }, - { id: "observability:opentelemetry", text: ["go.opentelemetry.io/otel"] }, - { id: "testing:testify+gomock", text: ["github.com/stretchr/testify", "go.uber.org/mock"] }, - { id: "forbidden:gin", forbiddenText: ["github.com/gin-gonic/gin"] }, - { id: "forbidden:echo", forbiddenText: ["github.com/labstack/echo"] }, - { id: "forbidden:fiber", forbiddenText: ["github.com/gofiber/fiber"] }, - { id: "forbidden:gorm", forbiddenText: ["gorm.io/gorm"] }, - { id: "forbidden:ent", forbiddenText: ["entgo.io/ent"] }, - { id: "forbidden:viper", forbiddenText: ["github.com/spf13/viper"] }, - { - id: "forbidden:redis", - forbiddenText: ["github.com/redis/go-redis", "github.com/go-redis/redis"], - }, - { id: "forbidden:zap", forbiddenText: ["go.uber.org/zap"] }, - ], - validationProfile: { packageManager: "bun", native: ["go"] }, -}; diff --git a/scripts/scaffbench/specs/python-ingestion-api.ts b/scripts/scaffbench/specs/python-ingestion-api.ts deleted file mode 100644 index 39558afce..000000000 --- a/scripts/scaffbench/specs/python-ingestion-api.ts +++ /dev/null @@ -1,121 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const PythonIngestionApiSpec: BenchmarkSpec = { - id: "python-ingestion-api", - introducedAt: "2026-08-21", - title: "Python FastAPI ingestion API with AI, queues, realtime, and quality gates", - lane: "core", - difficulty: 1, - family: "python", - supportedByBetterFullstack: true, - requirements: [ - "Create a Python API project using FastAPI for an AI document-ingestion pipeline: upload, queue, extract, and stream progress.", - "Use SQLModel for database models and Pydantic for validation.", - "Use LangGraph and OpenAI SDK for the extraction workflow.", - "Use JWT auth, Celery task queues, WebSockets for realtime job progress, Redis caching, OpenTelemetry, Typer, Rich, Ruff, Pytest, and Hypothesis.", - "Do not choose Django REST Framework, Django Ninja, or Flask, this is a FastAPI project.", - ], - naturalPrompt: - "Build a Python ingestion API starter for AI document processing. It needs FastAPI, SQL-backed models, strict validation, AI workflow libraries, queued workers, realtime job updates, Redis cache, tracing, CLI tools, and real test/quality tooling. Avoid Django-only API libraries.", - rightLibraryNotes: [ - "FastAPI is required; Django-specific API packages are forbidden.", - "SQLModel is required for the database layer.", - "LangGraph plus OpenAI SDK are required for AI workflow scaffolding.", - "Celery is required for background ingestion jobs.", - ], - canonicalFlags: [ - "--ecosystem", - "python", - "--database", - "postgres", - "--python-web-framework", - "fastapi", - "--python-orm", - "sqlmodel", - "--python-validation", - "pydantic", - "--python-ai", - "langgraph", - "openai-sdk", - "--python-auth", - "jwt", - "--python-api", - "none", - "--python-task-queue", - "celery", - "--python-graphql", - "none", - "--python-quality", - "ruff", - "--python-testing", - "pytest", - "hypothesis", - "--python-caching", - "redis", - "--python-realtime", - "websockets", - "--python-observability", - "opentelemetry", - "--python-cli", - "typer", - "rich", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "none", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "python", - database: "postgres", - pythonWebFramework: "fastapi", - pythonOrm: "sqlmodel", - pythonValidation: "pydantic", - pythonAi: ["langgraph", "openai-sdk"], - pythonAuth: "jwt", - pythonApi: "none", - pythonTaskQueue: "celery", - pythonQuality: "ruff", - pythonTesting: ["pytest", "hypothesis"], - pythonCaching: "redis", - pythonRealtime: "websockets", - pythonObservability: "opentelemetry", - pythonCli: ["typer", "rich"], - }, - strictMarkers: [ - { id: "backend:fastapi", text: ["fastapi"] }, - { id: "orm:sqlmodel", text: ["sqlmodel"] }, - { id: "validation:pydantic", text: ["pydantic"] }, - { id: "ai:langgraph", text: ["langgraph"] }, - { id: "ai:openai-sdk", text: ["openai"] }, - { id: "auth:jwt", text: ["jwt"] }, - { id: "jobs:celery", text: ["celery"] }, - { id: "quality:ruff", text: ["ruff"] }, - { id: "testing:pytest", text: ["pytest"] }, - { id: "testing:hypothesis", text: ["hypothesis"] }, - { id: "realtime:websockets", text: ["websockets"] }, - { id: "cli:typer+rich", text: ["typer", "rich"] }, - { id: "caching:redis", text: ["redis"] }, - { id: "observability:opentelemetry", textAny: ["opentelemetry"] }, - { - id: "forbidden:django-api", - forbiddenText: [ - "djangorestframework", - "django-rest-framework", - "rest_framework", - "django-ninja", - "django_ninja", - ], - }, - { id: "forbidden:flask", forbiddenText: ["flask"] }, - ], - validationProfile: { native: ["python"] }, -}; diff --git a/scripts/scaffbench/specs/react-native-expo.ts b/scripts/scaffbench/specs/react-native-expo.ts deleted file mode 100644 index 0f2d662c9..000000000 --- a/scripts/scaffbench/specs/react-native-expo.ts +++ /dev/null @@ -1,113 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const ReactNativeExpoSpec: BenchmarkSpec = { - id: "react-native-expo", - introducedAt: "2026-08-21", - title: "React Native Expo habit tracker with Expo Router, Uniwind, MMKV, and Maestro + RNTL", - lane: "core", - difficulty: 2, - family: "react-native", - supportedByBetterFullstack: true, - requirements: [ - "Create a React Native (Expo) habit-tracker app: a habit list, a detail screen, and a settings screen, all working offline.", - "Use Expo Router for navigation.", - "Use Uniwind for Tailwind-style styling. NativeWind is the familiar answer and the wrong one here, do not add it.", - "Use MMKV for on-device storage; all habit data lives on the device.", - "Use Maestro plus React Native Testing Library for testing.", - "Use Expo Notifications for habit reminders, Expo Updates for OTA, and Expo Linking for deep linking into a habit's detail screen.", - "This is a mobile-only project: no backend, database, sync service, or auth.", - ], - naturalPrompt: - "Build a React Native habit tracker on Expo that works fully offline: a habit list, detail and settings screens, local reminders, deep links into a habit, and over-the-air updates. It needs file-based navigation, Tailwind-style styling, fast on-device key-value storage, and both end-to-end and unit testing. It is a standalone mobile app with no server, database, or accounts.", - rightLibraryNotes: [ - "Expo Router is required for navigation.", - "Uniwind is the required styling approach (native-uniwind frontend); NativeWind is a failure.", - "MMKV is required for storage; Maestro + RNTL for testing.", - "Expo Notifications / Updates / Linking are the required push / OTA / deep-linking choices.", - ], - canonicalFlags: [ - "--ecosystem", - "react-native", - "--frontend", - "native-uniwind", - "--auth", - "none", - "--mobile-navigation", - "expo-router", - "--mobile-ui", - "uniwind", - "--mobile-storage", - "mmkv", - "--mobile-testing", - "maestro-react-native-testing-library", - "--mobile-push", - "expo-notifications", - "--mobile-ota", - "expo-updates", - "--mobile-deep-linking", - "expo-linking", - "--ai-docs", - "claude-md", - "--package-manager", - "bun", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "react-native", - frontend: ["native-uniwind"], - mobileNavigation: "expo-router", - mobileUI: "uniwind", - mobileStorage: "mmkv", - mobileTesting: "maestro-react-native-testing-library", - mobilePush: "expo-notifications", - mobileOTA: "expo-updates", - mobileDeepLinking: "expo-linking", - }, - strictMarkers: [ - { id: "nav:expo-router", deps: ["expo-router"] }, - { id: "styling:uniwind", deps: ["uniwind"] }, - { id: "storage:mmkv", deps: ["react-native-mmkv"] }, - { id: "push:expo-notifications", deps: ["expo-notifications"] }, - { id: "ota:expo-updates", deps: ["expo-updates"] }, - { id: "deep-linking:expo-linking", deps: ["expo-linking"] }, - { id: "testing:rntl", deps: ["@testing-library/react-native"] }, - { id: "testing:maestro", files: [".maestro/*.y*ml"] }, - { id: "forbidden:nativewind", forbiddenDeps: ["nativewind"] }, - { - id: "forbidden:backend", - forbiddenDeps: ["express", "hono", "fastify", "elysia", "@nestjs/core", "convex"], - }, - { - id: "forbidden:database", - forbiddenDeps: [ - "drizzle-orm", - "@prisma/client", - "@supabase/supabase-js", - "firebase", - "@react-native-firebase/app", - ], - }, - { - id: "forbidden:sync", - forbiddenDeps: [ - "@powersync/react-native", - "@instantdb/react-native", - "replicache", - "@rocicorp/zero", - "@nozbe/watermelondb", - ], - }, - { - id: "forbidden:auth", - forbiddenDeps: [ - "better-auth", - "@clerk/clerk-expo", - "expo-auth-session", - "@react-native-google-signin/google-signin", - ], - }, - ], - validationProfile: { packageManager: "bun" }, -}; diff --git a/scripts/scaffbench/specs/rust-leptos-axum.ts b/scripts/scaffbench/specs/rust-leptos-axum.ts deleted file mode 100644 index 091d9c55d..000000000 --- a/scripts/scaffbench/specs/rust-leptos-axum.ts +++ /dev/null @@ -1,148 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const RustLeptosAxumSpec: BenchmarkSpec = { - id: "rust-leptos-axum", - introducedAt: "2026-08-21", - title: "Rust Axum API with a Leptos WASM frontend and typed service libraries", - lane: "core", - difficulty: 1, - family: "rust", - supportedByBetterFullstack: true, - requirements: [ - "Create a Rust project for an internal feature-flag console with Axum as the backend web framework (not Actix).", - "Use Leptos specifically for the WASM frontend; do not replace it with Dioxus, Yew, or a JavaScript frontend.", - "Use PostgreSQL with SQLx.", - "Use Tonic for a typed API boundary.", - "Include Clap CLI support, tracing, anyhow/thiserror, Moka caching, OAuth2 auth, Lapin jobs, OpenTelemetry, and Askama templates.", - "Include serde, uuid, chrono, reqwest, config, utoipa, validator, and tokio-test libraries.", - ], - naturalPrompt: - "Build a Rust starter for an internal feature-flag console. It should have an Axum server, a Rust WASM frontend, Postgres access, typed service/API boundaries, CLI/admin utilities, tracing, auth, cache, jobs, and template rendering. Choose the right Rust libraries rather than swapping in web defaults.", - rightLibraryNotes: [ - "Leptos is required for the Rust WASM frontend.", - "Axum is required for the server.", - "SQLx is required for database access.", - "Tonic is required for typed RPC/API contracts.", - ], - canonicalFlags: [ - "--ecosystem", - "rust", - "--database", - "postgres", - "--rust-web-framework", - "axum", - "--rust-frontend", - "leptos", - "--rust-orm", - "sqlx", - "--rust-api", - "tonic", - "--rust-cli", - "clap", - "--rust-libraries", - "serde", - "uuid", - "chrono", - "reqwest", - "config", - "utoipa", - "validator", - "tokio-test", - "--rust-logging", - "tracing", - "--rust-error-handling", - "anyhow-thiserror", - "--rust-caching", - "moka", - "--rust-auth", - "oauth2", - "--rust-realtime", - "none", - "--rust-message-queue", - "lapin", - "--rust-observability", - "opentelemetry", - "--rust-templating", - "askama", - "--email", - "none", - "--observability", - "none", - "--caching", - "none", - "--search", - "none", - "--ai-docs", - "none", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "rust", - database: "postgres", - rustWebFramework: "axum", - rustFrontend: "leptos", - rustOrm: "sqlx", - rustApi: "tonic", - rustCli: "clap", - rustLibraries: [ - "serde", - "uuid", - "chrono", - "reqwest", - "config", - "utoipa", - "validator", - "tokio-test", - ], - rustLogging: "tracing", - rustErrorHandling: "anyhow-thiserror", - rustCaching: "moka", - rustAuth: "oauth2", - rustMessageQueue: "lapin", - rustObservability: "opentelemetry", - rustTemplating: "askama", - }, - strictMarkers: [ - { id: "rust:axum", text: ["axum"] }, - { - id: "frontend:leptos", - text: ["leptos", "leptos_router"], - files: ["*/Cargo.toml"], - }, - { id: "orm:sqlx", text: ["sqlx"] }, - { - id: "db:postgres", - textAny: ["PgPool", "sqlx::postgres", "sqlx::Postgres", "tokio-postgres"], - }, - { id: "api:tonic", text: ["tonic"] }, - { id: "cli:clap", text: ["clap"] }, - { id: "logging:tracing", text: ["tracing"] }, - { id: "cache:moka", text: ["moka"] }, - { id: "auth:oauth2", text: ["oauth2"] }, - { id: "jobs:lapin", text: ["lapin"] }, - { id: "observability:opentelemetry", text: ["opentelemetry"] }, - { id: "templating:askama", text: ["askama"] }, - { id: "errors:anyhow-thiserror", text: ["anyhow", "thiserror"] }, - { id: "lib:serde", text: ["serde"] }, - { id: "lib:uuid", text: ["uuid"] }, - { id: "lib:chrono", text: ["chrono"] }, - { id: "lib:reqwest", text: ["reqwest"] }, - { id: "lib:config", textAny: ['config = "', "config = {", "config.workspace"] }, - { id: "lib:utoipa", text: ["utoipa"] }, - { id: "lib:validator", text: ["validator"] }, - { id: "lib:tokio-test", text: ["tokio-test"] }, - { - id: "forbidden:dioxus", - forbiddenText: ["dioxus-router", "dioxus::prelude", "dioxus = {", "dioxus.workspace"], - forbiddenFiles: ["Dioxus.toml"], - }, - { id: "forbidden:actix", forbiddenText: ["actix-web"] }, - { - id: "forbidden:yew", - forbiddenText: ["yew::prelude", "yew = {", 'yew = "', "yew.workspace"], - }, - ], - validationProfile: { native: ["cargo"] }, -}; diff --git a/scripts/scaffbench/specs/ts-minimal-restraint.ts b/scripts/scaffbench/specs/ts-minimal-restraint.ts deleted file mode 100644 index 05430eaae..000000000 --- a/scripts/scaffbench/specs/ts-minimal-restraint.ts +++ /dev/null @@ -1,191 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const TsMinimalRestraintSpec: BenchmarkSpec = { - id: "ts-minimal-restraint", - introducedAt: "2026-08-21", - title: "Minimal React + Tailwind SPA with no backend, data, or auth (restraint test)", - lane: "extended", - difficulty: 1, - family: "typescript", - supportedByBetterFullstack: true, - requirements: [ - "Create a minimal TypeScript React single-page app built with Vite and Tailwind.", - "It is the launch page for a developer tool: hero, feature grid, pricing table, and a waitlist email form.", - "The waitlist form POSTs to an external form endpoint read from an env var, you do NOT build the endpoint.", - "Pricing is three static tiers, no checkout, no billing provider.", - "Do NOT add a backend, database, ORM, API layer, auth, payments, email, file storage, jobs, CMS, or analytics. Every one of those is tempting here; adding any of them is a failure.", - "Include Turborepo tooling.", - ], - naturalPrompt: - "Build the launch page for a developer tool as a React single-page app with Tailwind: hero, feature grid, static three-tier pricing, and a waitlist email form that posts to an external form service configured by env var. There are no accounts, no database, and no server of your own, keep it a lean front end and resist adding backend or data tooling.", - rightLibraryNotes: [ - "This is a frontend-only starter: do not add a backend, database, ORM, API, auth, payments, or email.", - "The pricing table and waitlist form are bait, they need no billing provider and no server.", - ], - canonicalFlags: [ - "--ecosystem", - "typescript", - "--frontend", - "react-vite", - "--backend", - "none", - "--runtime", - "none", - "--api", - "none", - "--database", - "none", - "--orm", - "none", - "--db-setup", - "none", - "--auth", - "none", - "--payments", - "none", - "--email", - "none", - "--file-upload", - "none", - "--logging", - "none", - "--observability", - "none", - "--feature-flags", - "none", - "--analytics", - "none", - "--effect", - "none", - "--state-management", - "none", - "--forms", - "none", - "--validation", - "none", - "--testing", - "none", - "--ai", - "none", - "--realtime", - "none", - "--job-queue", - "none", - "--animation", - "none", - "--css-framework", - "tailwind", - "--ui-library", - "none", - "--cms", - "none", - "--caching", - "none", - "--rate-limit", - "none", - "--i18n", - "none", - "--search", - "none", - "--vector-db", - "none", - "--file-storage", - "none", - "--web-deploy", - "none", - "--server-deploy", - "none", - "--addons", - "turborepo", - "--examples", - "none", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "typescript", - frontend: ["react-vite"], - backend: "none", - database: "none", - orm: "none", - api: "none", - auth: "none", - cssFramework: "tailwind", - }, - expectedAddons: ["turborepo"], - strictMarkers: [ - { id: "frontend:react-vite", deps: ["react", "vite"] }, - { id: "css:tailwind", deps: ["tailwindcss"] }, - { id: "tooling:turborepo", explicitOnly: true, deps: ["turbo"] }, - { - id: "forbidden:backend", - forbiddenDeps: ["hono", "express", "fastify", "elysia", "@nestjs/core"], - }, - { - id: "forbidden:database", - forbiddenDeps: ["drizzle-orm", "@prisma/client", "kysely", "typeorm", "mongoose"], - }, - { - id: "forbidden:auth", - forbiddenDeps: ["better-auth", "lucia", "next-auth", "@clerk/clerk-react"], - }, - { id: "forbidden:payments", forbiddenDeps: ["stripe", "@stripe/stripe-js", "polar-sh"] }, - { id: "forbidden:email", forbiddenDeps: ["resend", "nodemailer", "@react-email/components"] }, - { id: "forbidden:api", forbiddenDeps: ["@orpc/server", "@trpc/server", "graphql"] }, - { - id: "forbidden:jobs", - forbiddenDeps: [ - "inngest", - "bullmq", - "@trigger.dev/sdk", - "pg-boss", - "graphile-worker", - "@temporalio/", - ], - }, - { - id: "forbidden:cms", - forbiddenDeps: [ - "@sanity/client", - "sanity", - "next-sanity", - "contentful", - "payload", - "@payloadcms/", - "@keystone-6/core", - "@strapi/strapi", - "@strapi/client", - "@directus/sdk", - "@keystatic/", - "tinacms", - ], - }, - { - id: "forbidden:analytics", - forbiddenDeps: [ - "posthog-js", - "@vercel/analytics", - "@segment/analytics-next", - "plausible-tracker", - "mixpanel-browser", - ], - forbiddenText: ["googletagmanager", "gtag(", "umami"], - }, - { - id: "forbidden:file-storage", - forbiddenDeps: [ - "@aws-sdk/client-s3", - "uploadthing", - "@uploadthing/react", - "cloudinary", - "@supabase/storage-js", - ], - }, - ], - validationProfile: { packageManager: "bun" }, -}; diff --git a/scripts/scaffbench/specs/ts-svelte-edge-orpc.ts b/scripts/scaffbench/specs/ts-svelte-edge-orpc.ts deleted file mode 100644 index f90e2c5e6..000000000 --- a/scripts/scaffbench/specs/ts-svelte-edge-orpc.ts +++ /dev/null @@ -1,147 +0,0 @@ -import type { BenchmarkSpec } from "@scaffbench/types"; - -export const TsSvelteEdgeOrpcSpec: BenchmarkSpec = { - id: "ts-svelte-edge-orpc", - introducedAt: "2026-08-21", - title: "SvelteKit edge app on Cloudflare Workers with Hono + oRPC and D1", - lane: "core", - difficulty: 2, - family: "typescript", - supportedByBetterFullstack: true, - requirements: [ - "Create a TypeScript monorepo for an edge-deployed link-in-bio app: public pages render at the edge, signed-in users edit their page.", - "Use a SvelteKit web frontend styled with Tailwind.", - "Use a Hono backend running on the Cloudflare Workers runtime.", - "Use oRPC for the app API. tRPC is the obvious pick and the wrong one, it does not support a Svelte frontend.", - "Use SQLite via Cloudflare D1 with Drizzle as the ORM.", - "Use Better Auth for accounts and Valibot for validation. Every library must actually run on Workers, no Node-only APIs.", - "Deploy both the web app and the server to Cloudflare. The SvelteKit build must use the Cloudflare adapter; a Node server adapter is a failure.", - "Do not add payments, email, realtime, search, vector DB, jobs, CMS, file storage/upload, analytics, or i18n.", - ], - naturalPrompt: - "Build an edge-first starter that runs on Cloudflare. It needs a Svelte web app, a lightweight server on the Workers runtime, a type-safe app API, an edge SQL database with a typed ORM, account auth, and validation. Every choice has to actually run on Workers, resolve the conflicts that creates instead of reaching for Node defaults, and both apps deploy to Cloudflare.", - rightLibraryNotes: [ - "oRPC is required for the API because tRPC does not support a Svelte frontend.", - "The Workers runtime requires the Hono backend.", - "Cloudflare D1 is required for the SQLite database, and Cloudflare for deploys.", - "Better Auth must use a Workers-compatible ORM (Drizzle).", - "The SvelteKit build must target the Cloudflare adapter, not a Node server adapter.", - ], - canonicalFlags: [ - "--ecosystem", - "typescript", - "--frontend", - "svelte", - "--backend", - "hono", - "--runtime", - "workers", - "--api", - "orpc", - "--database", - "sqlite", - "--orm", - "drizzle", - "--db-setup", - "d1", - "--auth", - "better-auth", - "--validation", - "valibot", - "--css-framework", - "tailwind", - "--ui-library", - "none", - "--web-deploy", - "cloudflare", - "--server-deploy", - "cloudflare", - "--payments", - "none", - "--email", - "none", - "--file-upload", - "none", - "--file-storage", - "none", - "--logging", - "none", - "--observability", - "none", - "--feature-flags", - "none", - "--analytics", - "none", - "--effect", - "none", - "--state-management", - "none", - "--forms", - "none", - "--testing", - "none", - "--ai", - "none", - "--realtime", - "none", - "--job-queue", - "none", - "--animation", - "none", - "--cms", - "none", - "--caching", - "none", - "--rate-limit", - "none", - "--i18n", - "none", - "--search", - "none", - "--vector-db", - "none", - "--addons", - "turborepo", - "--examples", - "none", - "--ai-docs", - "none", - "--package-manager", - "bun", - "--no-install", - "--no-git", - "--disable-analytics", - ], - expectedConfig: { - ecosystem: "typescript", - frontend: ["svelte"], - backend: "hono", - runtime: "workers", - api: "orpc", - database: "sqlite", - orm: "drizzle", - dbSetup: "d1", - auth: "better-auth", - validation: "valibot", - cssFramework: "tailwind", - webDeploy: "cloudflare", - serverDeploy: "cloudflare", - }, - expectedAddons: ["turborepo"], - strictMarkers: [ - { id: "frontend:svelte", deps: ["@sveltejs/kit"] }, - { id: "backend:hono", deps: ["hono"] }, - { id: "api:orpc", deps: ["@orpc/server"], source: ["@orpc/"] }, - { id: "runtime:workers", deps: ["wrangler"] }, - { id: "orm:drizzle", deps: ["drizzle-orm"], source: ["drizzle-orm"] }, - { id: "db:d1", textAny: ["d1_databases", "D1Database", "d1-http"] }, - { id: "auth:better-auth", deps: ["better-auth"], source: ["better-auth"] }, - { id: "validation:valibot", deps: ["valibot"] }, - { id: "css:tailwind", deps: ["tailwindcss"] }, - { id: "adapter:cloudflare", deps: ["@sveltejs/adapter-cloudflare"] }, - { id: "forbidden:trpc", forbiddenDeps: ["@trpc/server", "@trpc/client"] }, - { id: "forbidden:next", forbiddenDeps: ["next"] }, - { id: "forbidden:adapter-node", forbiddenDeps: ["@sveltejs/adapter-node"] }, - ], - validationProfile: { packageManager: "bun" }, -}; diff --git a/scripts/scaffbench/summary.ts b/scripts/scaffbench/summary.ts deleted file mode 100644 index 7e1cd9023..000000000 --- a/scripts/scaffbench/summary.ts +++ /dev/null @@ -1,575 +0,0 @@ -import type { - BenchmarkSpec, - Effort, - PublicationEligibility, - RunOutcome, - RunResult, - ScaffbenchOptions, - ScaffbenchSummary, - SummaryAggregate, -} from "@scaffbench/types"; - -import { agentLabelForModel, providerForModel } from "@scaffbench/agents"; -import { - resolvedBfVersion, - SCAFFBENCH_SPEC_SCORE_WEIGHTS, - MIN_CI_RUNS, - MIN_RANKED_TRIALS, - HARNESS_VERSION, - PROMPT_VERSION, - SCAFFBENCH_SUITE_VERSION, - VALIDATION_CACHE_VERSION, - tryCommandText, -} from "@scaffbench/constants"; -import { - validationPassed, - qualityPassed, - classifyOutcome, - outcomeEvidenceFor, - scoredOutcome, - specScore, -} from "@scaffbench/scoring"; -import { specDifficulty } from "@scaffbench/specs"; -import { EVIDENCE_SCHEMA_VERSION } from "@scripts/verified-combinations/evidence"; -import * as Effect from "effect/Effect"; -import { existsSync } from "node:fs"; -import { rename, writeFile } from "node:fs/promises"; -import path from "node:path"; - -export function aggregateResults(results: readonly RunResult[]) { - const bySpecCell = aggregateBy(results, (result) => - [result.specId, result.model, result.effort, result.path].join("|"), - ); - const leaderboard = aggregateBy(results, (result) => - [result.model, result.effort, result.path].join("|"), - ); - for (const row of leaderboard) { - const scoredCells = bySpecCell.filter( - (cell) => - cell.model === row.model && - cell.effort === row.effort && - cell.path === row.path && - cell.scoredRuns > 0 && - cell.avgLines !== null, - ); - row.avgLines = nullableAverage(scoredCells.map((cell) => cell.avgLines!)); - } - return { bySpecCell, leaderboard }; -} - -function aggregateBy( - results: readonly RunResult[], - keyFor: (result: RunResult) => string, -): SummaryAggregate[] { - const groups = new Map(); - for (const result of results) { - const key = keyFor(result); - groups.set(key, [...(groups.get(key) ?? []), result]); - } - return [...groups.entries()] - .map(([key, group]) => { - const first = group[0]; - if (!first) throw new Error(`empty aggregate group: ${key}`); - const scored = group.filter(scoredOutcome); - const inconclusiveCount = group.length - scored.length; - const passCount = scored.filter(validationPassed).length; - const qualityMeasured = scored.filter((result) => qualityPassed(result) !== "na"); - const qualityPassCount = qualityMeasured.filter( - (result) => qualityPassed(result) === true, - ).length; - const ci = wilsonInterval(passCount, scored.length); - - const bySpec = new Map(); - for (const result of group) { - const entry = bySpec.get(result.specId) ?? { total: 0, scored: 0, pass: 0 }; - entry.total += 1; - if (scoredOutcome(result)) { - entry.scored += 1; - if (validationPassed(result)) entry.pass += 1; - } - bySpec.set(result.specId, entry); - } - const specEntries = [...bySpec.values()]; - const measuredSpecs = specEntries.filter((entry) => entry.scored > 0); - const macroPassRate = average( - measuredSpecs.map((entry) => (entry.pass / entry.scored) * 100), - ); - const passAllSpecs = specEntries.filter((entry) => entry.pass === entry.total).length; - const passAnySpecs = specEntries.filter((entry) => entry.pass > 0).length; - - const stackPercent = average(scored.map((result) => result.stackScore.percent)); - const commandDisciplinePercent = average( - scored.map((result) => - result.toolCompliance.total > 0 - ? Math.round((result.toolCompliance.score / result.toolCompliance.total) * 100) - : 0, - ), - ); - const scoresBySpec = new Map(); - for (const result of scored) { - const scores = scoresBySpec.get(result.specId) ?? []; - scoresBySpec.set(result.specId, [...scores, specScore(result).score]); - } - const weighted = [...scoresBySpec.entries()].reduce( - (acc, [specId, scores]) => { - const weight = specDifficulty(specId); - return { sum: acc.sum + weight * averagePrecise(scores), weight: acc.weight + weight }; - }, - { sum: 0, weight: 0 }, - ); - const index = weighted.weight > 0 ? Math.round((100 * weighted.sum) / weighted.weight) : 0; - const specScorePercent = Math.round( - 100 * averagePrecise(scored.map((result) => specScore(result).score)), - ); - const durations = group.map((result) => result.claude.durationMs); - - return { - key, - specId: key.startsWith(first.specId) ? first.specId : undefined, - model: first.model, - effort: first.effort, - effectiveReasoning: first.effectiveReasoning, - path: first.path, - runs: group.length, - scoredRuns: scored.length, - inconclusiveCount, - passCount, - passRate: scored.length > 0 ? Math.round((passCount / scored.length) * 100) : 0, - qualityPassCount, - qualityScoredRuns: qualityMeasured.length, - qualityPassRate: - qualityMeasured.length > 0 - ? Math.round((qualityPassCount / qualityMeasured.length) * 100) - : 0, - passCi95: ci, - ciReportable: scored.length >= MIN_CI_RUNS, - specCount: specEntries.length, - macroPassRate, - passAnySpecs, - passAllSpecs, - stackPercent, - faithfulnessPercent: maybeAverage( - scored.map((result) => result.generatorFaithfulness?.percent), - ), - acceptancePercent: maybeAverage(scored.map((result) => result.acceptanceScore?.percent)), - commandDisciplinePercent, - index, - specScore: specScorePercent, - avgDurationMs: average(durations), - medianDurationMs: percentile(durations, 50), - p95DurationMs: percentile(durations, 95), - avgOutputTokens: maybeAverage(group.map((result) => result.claude.outputTokens)), - avgCostUsd: maybeAveragePrecise(group.map((result) => result.claude.totalCostUsd)), - avgLines: nullableAverage( - scored.flatMap((result) => (result.codeMetrics ? [result.codeMetrics.lines] : [])), - ), - failureTags: countFailureTags(group), - outcomeCounts: countOutcomes(group), - publicationEligibility: publicationEligibility(group), - }; - }) - .sort( - (a, b) => - b.index - a.index || b.macroPassRate - a.macroPassRate || a.avgDurationMs - b.avgDurationMs, - ); -} - -function countOutcomes(group: readonly RunResult[]) { - const counts: Partial> = {}; - for (const result of group) { - const outcome = classifyOutcome(result); - counts[outcome] = (counts[outcome] ?? 0) + 1; - } - return counts; -} - -export function publicationEligibility(group: readonly RunResult[]): PublicationEligibility { - if (group.length === 0 || group.some((result) => !result.provenance)) return "exploratory"; - const signatures = new Set( - group.map((result) => { - const provenance = result.provenance!; - return JSON.stringify([ - provenance.suiteVersion, - provenance.harnessVersion, - provenance.validationCacheVersion, - provenance.promptVersion, - provenance.agentAdapter, - ]); - }), - ); - if (signatures.size !== 1) return "exploratory"; - const trialsBySpec = new Map>(); - for (const result of group) { - const trials = trialsBySpec.get(result.specId) ?? new Set(); - trials.add(result.trial); - trialsBySpec.set(result.specId, trials); - if ((result.provenance?.configuredTrials ?? 0) < MIN_RANKED_TRIALS) return "exploratory"; - } - return [...trialsBySpec.values()].every((trials) => trials.size >= MIN_RANKED_TRIALS) - ? "ranked" - : "exploratory"; -} - -function wilsonInterval(successes: number, total: number) { - if (total === 0) return { low: 0, high: 0 }; - const z = 1.96; - const p = successes / total; - const denom = 1 + (z * z) / total; - const center = p + (z * z) / (2 * total); - const margin = z * Math.sqrt((p * (1 - p) + (z * z) / (4 * total)) / total); - return { - low: Math.max(0, Math.round(((center - margin) / denom) * 100)), - high: Math.min(100, Math.round(((center + margin) / denom) * 100)), - }; -} - -function average(values: readonly number[]) { - if (values.length === 0) return 0; - return Math.round(averagePrecise(values)); -} - -function nullableAverage(values: readonly number[]) { - return values.length > 0 ? average(values) : null; -} - -function averagePrecise(values: readonly number[]) { - if (values.length === 0) return 0; - return values.reduce((sum, value) => sum + value, 0) / values.length; -} - -function percentile(values: readonly number[], p: number) { - if (values.length === 0) return 0; - const sorted = [...values].sort((a, b) => a - b); - const rank = Math.ceil((p / 100) * sorted.length); - const index = Math.min(sorted.length - 1, Math.max(0, rank - 1)); - return Math.round(sorted[index] ?? 0); -} - -function maybeAverage(values: readonly (number | undefined)[]) { - const present = values.filter((value): value is number => typeof value === "number"); - return present.length > 0 ? average(present) : undefined; -} - -function maybeAveragePrecise(values: readonly (number | undefined)[]) { - const present = values.filter((value): value is number => typeof value === "number"); - return present.length > 0 ? averagePrecise(present) : undefined; -} - -function countFailureTags(group: readonly RunResult[]) { - const counts: Record = {}; - for (const result of group) { - for (const tag of result.failureTags) { - counts[tag] = (counts[tag] ?? 0) + 1; - } - } - return counts; -} - -export async function writeSummary( - outDir: string, - results: readonly RunResult[], - options: ScaffbenchOptions, - specs: readonly BenchmarkSpec[], - metadata: Record, -) { - const { listSpecs, writeMatrixOnly, ...summaryOptions } = options; - void listSpecs; - void writeMatrixOnly; - const persistedResults = results.map((result) => ({ - ...result, - outcome: classifyOutcome(result), - outcomeEvidence: outcomeEvidenceFor(result), - })); - const aggregates = aggregateResults(persistedResults); - const summary: ScaffbenchSummary = { - harnessVersion: HARNESS_VERSION, - generatedAt: new Date().toISOString(), - options: summaryOptions, - metadata: { - ...metadata, - publicationEligibility: Object.fromEntries( - aggregates.leaderboard.map((row) => [row.key, row.publicationEligibility]), - ), - }, - specs: [...specs], - aggregates, - results: persistedResults, - }; - await writeAtomic(path.join(outDir, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`); - await writeAtomic(path.join(outDir, "summary.md"), renderMarkdown(summary)); -} - -async function writeAtomic(target: string, contents: string) { - const temporary = `${target}.tmp-${process.pid}`; - await writeFile(temporary, contents); - await rename(temporary, target); -} - -export function renderMarkdown(summary: ScaffbenchSummary) { - const rows = summary.results - .map((result) => - [ - result.specId, - result.trial, - result.effort, - result.effectiveReasoning ?? "", - result.model, - result.path, - formatOutcome(classifyOutcome(result)), - result.failureTags.join(", "), - result.claude.exitCode ?? "null", - formatSeconds(result.claude.durationMs), - result.claude.outputTokens ?? "", - result.claude.totalCostUsd?.toFixed(3) ?? "", - result.stackScore.percent, - `${result.stackScore.matched}/${result.stackScore.total}`, - result.generatorFaithfulness - ? `${result.generatorFaithfulness.matched}/${result.generatorFaithfulness.total}` - : "–", - result.acceptanceScore - ? `${result.acceptanceScore.matched}/${result.acceptanceScore.total}` - : "–", - result.validation.install?.exitCode ?? "", - result.validation.build?.exitCode ?? "", - result.validation.checkTypes?.exitCode ?? "", - result.validation.lint?.exitCode ?? "", - result.validation.test?.exitCode ?? "", - result.validation.deferred - ? "deferred" - : result.validation.cacheHit - ? "hit" - : result.validation.cacheKey - ? "miss" - : "", - ].join(" | "), - ) - .join("\n"); - - const aggregateRows = summary.aggregates.leaderboard - .map((aggregate) => - [ - aggregate.model, - aggregate.effort, - aggregate.effectiveReasoning ?? "", - aggregate.path, - aggregate.index, - `${aggregate.passCount}/${aggregate.scoredRuns}`, - aggregate.qualityScoredRuns > 0 - ? `${aggregate.qualityPassCount}/${aggregate.qualityScoredRuns}` - : "na", - aggregate.inconclusiveCount > 0 ? `${aggregate.inconclusiveCount}/${aggregate.runs}` : "0", - `${aggregate.macroPassRate}%`, - `${aggregate.passAnySpecs}/${aggregate.specCount}`, - `${aggregate.passAllSpecs}/${aggregate.specCount}`, - aggregate.ciReportable - ? `${aggregate.passRate}% (${aggregate.passCi95.low}-${aggregate.passCi95.high})` - : `n<${MIN_CI_RUNS}`, - `${aggregate.stackPercent}%`, - aggregate.faithfulnessPercent !== undefined ? `${aggregate.faithfulnessPercent}%` : "–", - aggregate.acceptancePercent !== undefined ? `${aggregate.acceptancePercent}%` : "–", - `${aggregate.commandDisciplinePercent}%`, - `${formatSeconds(aggregate.medianDurationMs)} / ${formatSeconds(aggregate.p95DurationMs)}`, - aggregate.avgOutputTokens ?? "", - aggregate.avgCostUsd?.toFixed(3) ?? "", - aggregate.publicationEligibility, - formatFailureTags(aggregate.failureTags), - ].join(" | "), - ) - .join("\n"); - - const cohortRows = cohortPassRates(summary.results, summary.specs) - .map( - (cohort) => - `| ${cohort.introducedAt} | ${cohort.specs} | ${cohort.passCount}/${cohort.scoredRuns} | ${cohort.passRate}% |`, - ) - .join("\n"); - - return `# ScaffBench ${SCAFFBENCH_SUITE_VERSION} Run - -Harness: ${summary.harnessVersion} -Agent: ${agentLabelForModel(summary.options.model)} (single agent; single model family per row) -Specs: ${summary.specs.map((spec) => spec.id).join(", ")} -Repeats: ${summary.options.repeats} -Prompt style: ${summary.options.promptStyle} - -## Path × effort summary - -This is an ablation across creation paths and reasoning effort for one agent -(${agentLabelForModel(summary.options.model)}), not a cross-vendor leaderboard. Pass rate is over *scored* runs. -Provider, harness, and validation infrastructure outcomes are inconclusive and -excluded; budget and generation-deadline exhaustion remain scored failures. - -"Pass@1" is the CORE pass rate, install + build + typecheck + native compile, -i.e. does the project actually build and run. "Quality" is the stricter advisory -tier (core + lint/format; tests, doctor and route run and are reported but -affect no score): a project can be Pass@1-green but Quality-red because it is -mis-formatted or a style-lint warns. Formatting is a quality metric, never a -brokenness verdict, so it does not move Pass@1. "Wired -libs" is scored from the generated artifact (deps + imports + files); -"Faithful" is the assisted-path bts.jsonc-vs-requested diagnostic. - -Reliability is reported per spec, not pooled: "Macro" is the mean of per-spec -pass rates; "pass@k" counts specs solved on at least one repeat and "pass^k" -specs solved on every repeat. The Wilson "CI95" is shown only when a cell has -≥ ${MIN_CI_RUNS} scored runs (below that it reads \`n<${MIN_CI_RUNS}\`, since e.g. 3/3 and 0/3 -intervals overlap and the interval is not informative). - -"Index" is the single rankable 0-100 composite the table is sorted by. Each -spec earns a graded score: ${Math.round(SCAFFBENCH_SPEC_SCORE_WEIGHTS.core * 100)}% for a Core pass, ${Math.round(SCAFFBENCH_SPEC_SCORE_WEIGHTS.quality * 100)}% for the share of lint and -format gates green (tests are not scored), and ${Math.round(SCAFFBENCH_SPEC_SCORE_WEIGHTS.stack * 100)}% for the stack score (wired libs, -traps, restraint). The index is the difficulty-weighted mean of those per-spec -scores (spec difficulty 1, 2, or 3 is pinned in the spec file), times 100. -Latency is median / p95 (wall-clock -moves with provider load, so the mean alone is misleading over small samples). - -| Model | Effort | Effective reasoning | Path | Index | Pass@1 | Quality | Inconclusive | Macro | pass@k | pass^k | CI95 | Wired libs | Faithful | Acceptance | Command discipline | Median / p95 | Avg output tokens | Avg cost | Publication | Failure tags | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | -${aggregateRows} - -## Introduction cohorts - -| Introduced | Specs | Pass@1 | Pass rate | -| --- | ---: | ---: | ---: | -${cohortRows} - -## Runs - -| Spec | Trial | Effort | Effective reasoning | Model | Path | Validation | Failure tags | Claude exit | Time | Output tokens | Cost | Wired % | Wired | Faithful | Acceptance | Install | Build | Typecheck | Lint | Test | Validation cache | -| --- | ---: | --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | -${rows} -`; -} - -function formatOutcome(outcome: RunOutcome) { - if (outcome === "success") return "pass"; - return outcome; -} - -export function cohortPassRates(results: readonly RunResult[], specs: readonly BenchmarkSpec[]) { - const introducedBySpec = new Map(specs.map((spec) => [spec.id, spec.introducedAt])); - const cohorts = new Map< - string, - { introducedAt: string; specIds: Set; scoredRuns: number; passCount: number } - >(); - for (const result of results) { - const introducedAt = introducedBySpec.get(result.specId) ?? "unknown"; - const cohort = cohorts.get(introducedAt) ?? { - introducedAt, - specIds: new Set(), - scoredRuns: 0, - passCount: 0, - }; - cohort.specIds.add(result.specId); - if (scoredOutcome(result)) { - cohort.scoredRuns += 1; - if (validationPassed(result)) cohort.passCount += 1; - } - cohorts.set(introducedAt, cohort); - } - return [...cohorts.values()] - .map((cohort) => ({ - introducedAt: cohort.introducedAt, - specs: cohort.specIds.size, - scoredRuns: cohort.scoredRuns, - passCount: cohort.passCount, - passRate: - cohort.scoredRuns > 0 ? Math.round((cohort.passCount / cohort.scoredRuns) * 100) : 0, - })) - .sort((a, b) => a.introducedAt.localeCompare(b.introducedAt)); -} - -function formatFailureTags(tags: Record) { - const entries = Object.entries(tags); - if (entries.length === 0) return ""; - return entries.map(([tag, count]) => `${tag}:${count}`).join(", "); -} - -function formatSeconds(ms: number) { - return `${(ms / 1000).toFixed(1)}s`; -} - -export function collectMetadata(options: ScaffbenchOptions) { - return Effect.gen(function* () { - const gitHead = yield* tryCommandText("git", ["rev-parse", "HEAD"], process.cwd()); - const gitBranch = yield* tryCommandText("git", ["branch", "--show-current"], process.cwd()); - const gitStatus = yield* tryCommandText("git", ["status", "--porcelain"], process.cwd()); - const bunVersion = yield* tryCommandText( - existsSync(`${process.env.HOME}/.bun/bin/bun`) ? `${process.env.HOME}/.bun/bin/bun` : "bun", - ["--version"], - process.cwd(), - ); - const bfGeneratorVersion = resolvedBfVersion() === "latest" ? undefined : resolvedBfVersion(); - const toolchains = yield* collectToolchainVersions(); - return { - cwd: process.cwd(), - evidenceSchemaVersion: EVIDENCE_SCHEMA_VERSION, - workspaceClean: gitStatus === "", - gitHead, - gitBranch, - bunVersion, - nodeVersion: process.version, - platform: process.platform, - arch: process.arch, - environmentQualified: true, - toolchains, - suiteVersion: SCAFFBENCH_SUITE_VERSION, - harnessVersion: HARNESS_VERSION, - validationCacheVersion: VALIDATION_CACHE_VERSION, - promptVersion: PROMPT_VERSION, - agentAdapter: providerForModel(options.model), - configuredTrials: options.repeats, - bfGeneratorVersion, - model: options.model, - effectiveReasoning: options.efforts.map((effort) => ({ - effort, - effectiveReasoning: effectiveReasoning(options.model, effort), - })), - }; - }); -} - -let toolchainVersionsMemo: Record | undefined; - -export function collectToolchainVersions() { - const bunBin = existsSync(`${process.env.HOME}/.bun/bin/bun`) - ? `${process.env.HOME}/.bun/bin/bun` - : "bun"; - const probes: Record = { - bun: [bunBin, ["--version"]], - node: ["node", ["--version"]], - rustc: ["rustc", ["--version"]], - cargo: ["cargo", ["--version"]], - go: ["go", ["version"]], - dotnet: ["dotnet", ["--version"]], - python: ["python3", ["--version"]], - uv: ["uv", ["--version"]], - java: ["java", ["--version"]], - mvn: ["mvn", ["-v"]], - mix: ["mix", ["--version"]], - buf: ["buf", ["--version"]], - protoc: ["protoc", ["--version"]], - psql: ["psql", ["--version"]], - }; - return Effect.gen(function* () { - if (toolchainVersionsMemo) return toolchainVersionsMemo; - const entries = yield* Effect.forEach( - Object.entries(probes), - ([name, [command, args]]) => - Effect.gen(function* () { - const version = yield* tryCommandText(command, [...args], process.cwd()); - return [name, version] as const; - }), - { concurrency: "unbounded" }, - ); - toolchainVersionsMemo = Object.fromEntries(entries); - return toolchainVersionsMemo; - }); -} - -export function effectiveReasoning(model: string, effort: Effort) { - if (effort !== "default") return effort; - const normalized = model.toLowerCase(); - if (normalized.includes("4-7") || normalized.includes("4.7")) return "xhigh"; - if (normalized.includes("4-6") || normalized.includes("4.6")) return "high"; - return undefined; -} diff --git a/scripts/scaffbench/tsconfig.json b/scripts/scaffbench/tsconfig.json deleted file mode 100644 index 5503d3ecd..000000000 --- a/scripts/scaffbench/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "baseUrl": "..", - "paths": { - "@scaffbench/*": ["./scaffbench/*"], - "@scripts/*": ["./*"], - "@testing/*": ["../testing/*"] - } - }, - "include": [ - "./**/*.ts", - "../scaffbench-v2.ts", - "../scaffbench-v2-lib.test.ts", - "../scaffbench-hardening.test.ts", - "../scaffbench-hardening-round-2.test.ts", - "../build-scaffbench-3-data.ts", - "../scaffbench-executor.test.ts" - ] -} diff --git a/scripts/scaffbench/types.ts b/scripts/scaffbench/types.ts deleted file mode 100644 index 679b288bb..000000000 --- a/scripts/scaffbench/types.ts +++ /dev/null @@ -1,345 +0,0 @@ -export type CreationPath = "prompt" | "mcp"; -export type Effort = "default" | "low" | "medium" | "high" | "xhigh" | "max"; -export type PromptStyle = "explicit" | "natural"; -export type CommandStatus = "pass" | "fail" | "unknown" | "skipped"; -export type FailureTag = - | "claude-error" - | "claude-timeout" - | "command-discipline" - | "doctor-failed" - | "format-failed" - | "install-failed" - | "lint-failed" - | "project-not-found" - | "route-failed" - | "stack-mismatch" - | "test-failed" - | "tool-violation" - | "typecheck-failed" - | "validation-failed" - | "build-failed" - | "budget-exhausted" - | "deadline-exhausted" - | "timeout-progressing" - | "timeout-stuck" - | "provider-infra" - | "harness-infra" - | "validation-infra" - | "toolchain-missing" - | "stack-unwired" - | "validation-deferred"; - -export type RunOutcome = - | "success" - | "skipped" - | "model-failure" - | "provider-infra" - | "harness-infra" - | "validation-infra" - | "budget-exhausted" - | "deadline-exhausted"; - -export type RunOutcomeRollup = "success" | "model-failure" | "infra-inconclusive"; -export type TimeoutProgress = "timeout-progressing" | "timeout-stuck"; -export type PublicationEligibility = "ranked" | "exploratory"; - -export type StrictMarker = { - id: string; - explicitOnly?: boolean; - deps?: readonly string[]; - source?: readonly string[]; - text?: readonly string[]; - textAny?: readonly string[]; - files?: readonly string[]; - forbiddenDeps?: readonly string[]; - forbiddenText?: readonly string[]; - forbiddenFiles?: readonly string[]; -}; - -export type PrerequisiteCommand = { - command: readonly string[]; - whenConfigFound?: readonly string[]; -}; - -export type SpecDifficulty = 1 | 2 | 3; - -export type BenchmarkSpec = { - id: string; - introducedAt: string; - title: string; - lane: "core" | "extended"; - difficulty: SpecDifficulty; - family: - | "typescript" - | "rust" - | "python" - | "go" - | "dotnet" - | "java" - | "elixir" - | "react-native" - | "multi-ecosystem"; - supportedByBetterFullstack: boolean; - paths?: readonly CreationPath[]; - requirements: readonly string[]; - naturalPrompt: string; - rightLibraryNotes: readonly string[]; - canonicalFlags: readonly string[]; - expectedConfig?: Record; - expectedParts?: readonly string[]; - expectedAddons?: readonly string[]; - strictMarkers: readonly StrictMarker[]; - acceptanceSets?: Record; - timeoutMultiplier?: number; - prerequisiteCommands?: readonly PrerequisiteCommand[]; - validationProfile: { - packageManager?: "bun"; - native?: readonly ("cargo" | "dotnet" | "go" | "python" | "java" | "elixir")[]; - qualityGate?: boolean; - doctorCheck?: boolean; - routeCheckCandidate?: boolean; - }; -}; - -export type StepResult = { - command: string; - exitCode: number | null; - timedOut: boolean; - spawnError?: boolean; - spawnErrorCode?: string; - transientNetwork?: boolean; - retryCount?: number; - status?: "ran" | "skip" | "na"; - durationMs: number; - stdoutTail: string; - stderrTail: string; -}; - -export type CommandResult = StepResult & { - stdout: string; - stderr: string; - timeoutKind?: "hard" | "idle"; - timeoutProgress?: TimeoutProgress; - startedAtMs?: number; - lastActivityAtMs?: number; - lastStdoutActivityAtMs?: number; - lastStderrActivityAtMs?: number; - lastProgressActivityAtMs?: number; -}; - -export type CommandDisciplineCheck = { - id: string; - status: CommandStatus; - detail: string; -}; - -export type ToolCompliance = { - score: number; - total: number; - checks: CommandDisciplineCheck[]; -}; - -export type ProjectValidation = { - projectExists: boolean; - qualityGateRequested?: boolean; - deferred?: boolean; - skipped?: boolean; - sourceHash?: string; - cacheKey?: string; - cacheHit?: boolean; - steps: Record; - install?: StepResult; - build?: StepResult; - checkTypes?: StepResult; - lint?: StepResult; - format?: StepResult; - test?: StepResult; - doctor?: StepResult; - route?: StepResult; -}; - -export type AgentRunAccounting = { - exitCode: number | null; - timedOut: boolean; - durationMs: number; - resultDurationMs?: number; - outputTokens?: number; - totalCostUsd?: number; - sessionId?: string; - terminalReason?: string; - spawnError?: boolean; - spawnErrorCode?: string; - timeoutKind?: "hard" | "idle"; - timeoutProgress?: TimeoutProgress; - stderrTail?: string; -}; - -export type BudgetPolicy = { - budgetEnforced: boolean; - maxBudgetUsd: number; -}; - -export type RunProvenance = { - suiteVersion: string; - harnessVersion: string; - validationCacheVersion: number; - promptVersion: string; - resourceProfileId?: string; - agentAdapter: string; - configuredTrials: number; - specOrderSeed: number; -}; - -export type TopUpRecord = { - trials: number; - specs: string[]; - recordedAt: string; -}; - -export type RunProtocol = { - repeats: number; - seed: number; - topUps?: TopUpRecord[]; -}; - -export type OutcomeEvidence = { - budgetEstimated?: true; -}; - -export type RepairResult = { - attemptedAt: string; - failingStep: string; - prompt: string; - claude: AgentRunAccounting; - validation: ProjectValidation; - stackScore: StackScore; - outcome: RunOutcome; - outcomeEvidence?: OutcomeEvidence; - failureTags: FailureTag[]; -}; - -export type StackScore = { - matched: number; - total: number; - percent: number; - misses: string[]; -}; - -export type CodeMetrics = { - files: number; - lines: number; - bytes: number; -}; - -export type RunResult = { - id: string; - specId: string; - specTitle: string; - model: string; - effort: Effort; - effectiveReasoning?: string; - path: CreationPath; - trial: number; - promptStyle: PromptStyle; - runDir: string; - projectName: string; - projectDir: string | null; - codeMetrics?: CodeMetrics; - claude: AgentRunAccounting; - outcome?: RunOutcome; - outcomeEvidence?: OutcomeEvidence; - budgetPolicy?: BudgetPolicy; - provenance?: RunProvenance; - validation: ProjectValidation; - stackScore: StackScore; - generatorFaithfulness?: StackScore; - acceptanceScore?: StackScore; - toolCompliance: ToolCompliance; - failureTags: FailureTag[]; - repair?: RepairResult; -}; - -export type ScaffbenchOptions = { - command?: "run" | "calibrate"; - model: string; - efforts: Effort[]; - paths: CreationPath[]; - specs: string[]; - specsExplicit?: boolean; - repeats: number; - topUp?: number; - outDir: string; - maxBudgetUsd: string; - skipValidation: boolean; - generateOnly: boolean; - validateExisting: boolean; - forceRevalidate: boolean; - qualityGate: boolean; - noQualityGate?: boolean; - doctorCheck: boolean; - routeCheck: boolean; - promptStyle: PromptStyle; - listSpecs: boolean; - writeMatrixOnly: boolean; - repair?: boolean; -}; - -export type SummaryAggregate = { - key: string; - specId?: string; - model: string; - effort: Effort; - effectiveReasoning?: string; - path: CreationPath; - runs: number; - scoredRuns: number; - inconclusiveCount: number; - passCount: number; - passRate: number; - qualityPassCount: number; - qualityScoredRuns: number; - qualityPassRate: number; - passCi95: { low: number; high: number }; - ciReportable: boolean; - specCount: number; - macroPassRate: number; - passAnySpecs: number; - passAllSpecs: number; - stackPercent: number; - faithfulnessPercent?: number; - acceptancePercent?: number; - commandDisciplinePercent: number; - index: number; - specScore: number; - avgDurationMs: number; - medianDurationMs: number; - p95DurationMs: number; - avgOutputTokens?: number; - avgCostUsd?: number; - avgLines: number | null; - failureTags: Record; - outcomeCounts: Partial>; - publicationEligibility: PublicationEligibility; -}; - -export type ScaffbenchSummary = { - harnessVersion: string; - generatedAt: string; - options: Omit; - metadata: Record; - specs: BenchmarkSpec[]; - aggregates: { - bySpecCell: SummaryAggregate[]; - leaderboard: SummaryAggregate[]; - }; - results: RunResult[]; -}; - -export type ProjectIndex = { - dependencies: Set; - files: Set; - packageText: string; - sourceText: string; - configText: string; - allText: string; -}; diff --git a/scripts/scaffbench/validation/bun.ts b/scripts/scaffbench/validation/bun.ts deleted file mode 100644 index 51c627e9e..000000000 --- a/scripts/scaffbench/validation/bun.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateBunProject } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/cache.ts b/scripts/scaffbench/validation/cache.ts deleted file mode 100644 index 3585c8239..000000000 --- a/scripts/scaffbench/validation/cache.ts +++ /dev/null @@ -1,232 +0,0 @@ -import type { BenchmarkSpec, ProjectValidation, ScaffbenchOptions } from "@scaffbench/types"; - -import * as FileSystem from "@effect/platform/FileSystem"; -import { - HARNESS_VERSION, - VALIDATION_CACHE_VERSION, - VALIDATION_RESOURCE_PROFILE_ID, -} from "@scaffbench/constants"; -import { collectToolchainVersions } from "@scaffbench/summary"; -import { - hasTransientNetworkSignature, - isRecurringTransientFailure, -} from "@scaffbench/validation/classification"; -import { effectiveValidationOptions, validateProject } from "@scaffbench/validation/index"; -import * as Effect from "effect/Effect"; -import { spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; -import { cp, lstat, readFile, readdir, readlink, rm } from "node:fs/promises"; -import path from "node:path"; - -const HASH_SKIP_DIRECTORIES = new Set([ - "node_modules", - ".git", - "dist", - "build", - ".next", - ".turbo", - "coverage", - "target", - ".venv", - "bin", - "obj", - "deps", - "_build", -]); - -export function validateProjectCached( - spec: BenchmarkSpec, - projectDir: string, - options: ScaffbenchOptions, -) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const sourceHash = yield* hashProjectSource(projectDir); - const toolchains = yield* collectToolchainVersions(); - const cacheKey = validationCacheKey(spec, options, sourceHash, toolchains); - const cacheDir = path.join(options.outDir, "validation-cache"); - const cachePath = path.join(cacheDir, `${cacheKey}.json`); - - if (!options.forceRevalidate) { - const cached = yield* readCachedValidation(cachePath, sourceHash, cacheKey); - if (cached) return cached; - } - - const cloneDir = `${projectDir}.validate-tmp`; - yield* Effect.tryPromise(() => rm(cloneDir, { recursive: true, force: true })); - yield* Effect.tryPromise(() => cloneProjectTree(projectDir, cloneDir)); - const validation = yield* validateProject(spec, cloneDir, options).pipe( - Effect.ensuring( - Effect.tryPromise(() => rm(cloneDir, { recursive: true, force: true })).pipe(Effect.ignore), - ), - ); - const withCacheMeta: ProjectValidation = { - ...validation, - sourceHash, - cacheKey, - cacheHit: false, - deferred: false, - }; - - if (cacheableValidation(withCacheMeta)) { - yield* fs.makeDirectory(cacheDir, { recursive: true }); - yield* fs.writeFileString( - cachePath, - `${JSON.stringify( - { - version: VALIDATION_CACHE_VERSION, - createdAt: new Date().toISOString(), - specId: spec.id, - validation: withCacheMeta, - }, - null, - 2, - )}\n`, - ); - } else { - yield* Effect.tryPromise(() => rm(cachePath, { force: true })).pipe(Effect.ignore); - } - return withCacheMeta; - }); -} - -function readCachedValidation(cachePath: string, sourceHash: string, cacheKey: string) { - return Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - if (!(yield* fs.exists(cachePath))) return null; - const text = yield* fs.readFileString(cachePath); - const cached = yield* Effect.try({ try: () => JSON.parse(text), catch: () => null }); - if (!cached?.validation?.projectExists) return null; - return { - ...cached.validation, - qualityGateRequested: cached.validation.qualityGateRequested === true, - sourceHash, - cacheKey, - cacheHit: true, - deferred: false, - } as ProjectValidation; - }).pipe(Effect.catchAll(() => Effect.succeed(null))); -} - -export function cacheableValidation(validation: ProjectValidation) { - return !Object.entries(validation.steps).some( - ([name, step]) => - step?.timedOut || - step?.spawnError || - (step !== undefined && - step.exitCode !== 0 && - (isRecurringTransientFailure(name, step) || hasTransientNetworkSignature(step))), - ); -} - -export function validationCacheKey( - spec: BenchmarkSpec, - options: ScaffbenchOptions, - sourceHash: string, - toolchains: Record, - environment: { platform: string; arch: string } = { - platform: process.platform, - arch: process.arch, - }, -) { - const toolchainHash = createHash("sha256") - .update(JSON.stringify(Object.entries(toolchains).sort(([a], [b]) => a.localeCompare(b)))) - .digest("hex"); - const effective = effectiveValidationOptions(spec, options); - const hash = createHash("sha256"); - hash.update( - JSON.stringify({ - version: VALIDATION_CACHE_VERSION, - harnessVersion: HARNESS_VERSION, - resourceProfileId: VALIDATION_RESOURCE_PROFILE_ID, - specId: spec.id, - sourceHash, - qualityGate: effective.qualityGate, - doctorCheck: effective.doctorCheck, - routeCheck: effective.routeCheck, - platform: environment.platform, - arch: environment.arch, - toolchainHash, - }), - ); - return hash.digest("hex"); -} - -async function cloneProjectTree(sourceDir: string, destDir: string) { - if (process.platform === "darwin") { - const clone = spawnSync("cp", ["-Rc", sourceDir, destDir], { stdio: "ignore" }); - if (clone.status === 0) return; - await rm(destDir, { recursive: true, force: true }); - } - await cp(sourceDir, destDir, { recursive: true, force: true, verbatimSymlinks: true }); -} - -type HashEntry = { - path: string; - kind: "file" | "symlink" | "directory"; - mode: number; - target?: string; -}; - -export function hashProjectSource(projectDir: string) { - return Effect.gen(function* () { - const hash = createHash("sha256"); - const entries = yield* listHashableEntries(projectDir); - yield* Effect.forEach( - entries, - (entry) => - Effect.gen(function* () { - const relative = path.relative(projectDir, entry.path).split(path.sep).join("/"); - hash.update(entry.kind); - hash.update("\0"); - hash.update(relative); - hash.update("\0"); - hash.update(entry.mode.toString(8)); - hash.update("\0"); - if (entry.kind === "file") - hash.update(yield* Effect.tryPromise(() => readFile(entry.path))); - if (entry.kind === "symlink") hash.update(entry.target ?? ""); - hash.update("\0"); - }), - { concurrency: 1, discard: true }, - ); - return hash.digest("hex"); - }); -} - -function listHashableEntries(root: string) { - return Effect.gen(function* () { - const entries: HashEntry[] = []; - - const visit = (directory: string): Effect.Effect => - Effect.gen(function* () { - const children = yield* Effect.tryPromise(() => - readdir(directory, { withFileTypes: true }), - ); - yield* Effect.forEach( - children, - (entry) => { - if (HASH_SKIP_DIRECTORIES.has(entry.name)) return Effect.void; - const entryPath = path.join(directory, entry.name); - return Effect.gen(function* () { - const info = yield* Effect.tryPromise(() => lstat(entryPath)); - const mode = info.mode & 0o7777; - if (entry.isSymbolicLink()) { - const target = yield* Effect.tryPromise(() => readlink(entryPath)); - entries.push({ path: entryPath, kind: "symlink", mode, target }); - } else if (entry.isDirectory()) { - entries.push({ path: entryPath, kind: "directory", mode }); - yield* visit(entryPath); - } else if (entry.isFile()) { - entries.push({ path: entryPath, kind: "file", mode }); - } - }); - }, - { concurrency: 1, discard: true }, - ); - }); - - yield* visit(root); - return entries.sort((a, b) => a.path.localeCompare(b.path)); - }); -} diff --git a/scripts/scaffbench/validation/cargo.ts b/scripts/scaffbench/validation/cargo.ts deleted file mode 100644 index 5cc2107ca..000000000 --- a/scripts/scaffbench/validation/cargo.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateCargoProject } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/classification.ts b/scripts/scaffbench/validation/classification.ts deleted file mode 100644 index e645abc8b..000000000 --- a/scripts/scaffbench/validation/classification.ts +++ /dev/null @@ -1,56 +0,0 @@ -import type { StepResult } from "@scaffbench/types"; - -const TRANSIENT_NETWORK_PATTERNS = [ - /\bEAI_AGAIN\b/i, - /\bENOTFOUND\b/i, - /\bETIMEDOUT\b/i, - /\bECONNRESET\b/i, - /TLS (?:handshake|connection).*?(?:failed|error|timeout)|handshake failure/i, -] as const; - -const REGISTRY = - /registry\.npmjs\.org|registry\.yarnpkg\.com|crates\.io|pypi\.org|files\.pythonhosted\.org|proxy\.golang\.org|nuget\.org|repo(?:1)?\.maven\.org|hex\.pm/i; -const HTTP_TRANSIENT_STATUS = - /(?:\bHTTP(?:\/\d(?:\.\d)?)?|\bstatus(?:\s+code)?|\berror)\D{0,10}(?:429|5\d\d)\b/i; -const PACKAGE_FETCH = - /\b(?:npm|bun|yarn|pnpm|cargo|crates?|pip|uv|poetry|pypi|golang|nuget|dotnet|maven|gradle|hex|mix)\b[^\n]{0,80}\b(?:fetch|download|registry|package|dependenc(?:y|ies)|install|request)|\b(?:fetch|download|registry|package|dependenc(?:y|ies)|install|request)\b[^\n]{0,80}\b(?:npm|bun|yarn|pnpm|cargo|crates?|pip|uv|poetry|pypi|golang|nuget|dotnet|maven|gradle|hex|mix)\b/i; -const MODEL_OWNED_NOT_FOUND = - /\b404\b|not found in (?:the )?registry|no matching version|does not exist/i; - -export function validationStepText(step: Pick) { - return `${step.stdoutTail ?? ""}\n${step.stderrTail ?? ""}`; -} - -/** Narrow transient set; a registry 404/nonexistent version deliberately misses. */ -export function hasTransientNetworkSignature(step: Pick) { - const lines = validationStepText(step).split(/\r?\n/); - const modelOwnedLines = new Set(); - const transientLines = new Set(); - for (const [index, line] of lines.entries()) { - if (MODEL_OWNED_NOT_FOUND.test(line)) modelOwnedLines.add(index); - const fetchContext = REGISTRY.test(line) || PACKAGE_FETCH.test(line); - if ( - TRANSIENT_NETWORK_PATTERNS.some((pattern) => pattern.test(line)) || - (fetchContext && (HTTP_TRANSIENT_STATUS.test(line) || /too many requests/i.test(line))) - ) { - transientLines.add(index); - } - } - // A missing package/version remains model-owned unless a distinct diagnostic - // line proves a separate transient failure (for example 404 followed by 503). - return [...transientLines].some((index) => !modelOwnedLines.has(index)); -} - -export function isInstallClassStep(stepName: string) { - const base = stepName.slice(stepName.lastIndexOf(":") + 1).toLowerCase(); - return /install|restore|download|deps(?:get)?|sync/.test(base); -} - -export function isRecurringTransientFailure(name: string, step: StepResult) { - return ( - step.exitCode !== 0 && - isInstallClassStep(name) && - (step.retryCount ?? 0) > 0 && - step.transientNetwork === true - ); -} diff --git a/scripts/scaffbench/validation/dotnet.ts b/scripts/scaffbench/validation/dotnet.ts deleted file mode 100644 index 445a11448..000000000 --- a/scripts/scaffbench/validation/dotnet.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateDotnetProject } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/elixir.ts b/scripts/scaffbench/validation/elixir.ts deleted file mode 100644 index 1af0caeaa..000000000 --- a/scripts/scaffbench/validation/elixir.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateElixirProject } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/executor.ts b/scripts/scaffbench/validation/executor.ts deleted file mode 100644 index 1fc2b32d4..000000000 --- a/scripts/scaffbench/validation/executor.ts +++ /dev/null @@ -1,136 +0,0 @@ -import type { CommandResult } from "@scaffbench/types"; - -import { quoteArg, tail } from "@scaffbench/agents/command"; -import { - VALIDATION_ENV_SCRUB_PATTERN, - VALIDATION_OUTPUT_LIMIT_BYTES, - VALIDATION_RESOURCE_ENV, -} from "@scaffbench/constants"; -import { spawnProcessTree } from "@scaffbench/process-tree"; -import { existsSync } from "node:fs"; - -const TASKPOLICY = "/usr/sbin/taskpolicy"; -const useTaskpolicy = process.platform === "darwin" && existsSync(TASKPOLICY); -const RETAINED_OUTPUT_CHARS = 262_144; - -export function validationEnv(extra?: Record): Record { - const env: Record = {}; - for (const [key, value] of Object.entries(process.env)) { - if (value === undefined || VALIDATION_ENV_SCRUB_PATTERN.test(key)) continue; - env[key] = value; - } - return { ...env, ...VALIDATION_RESOURCE_ENV, ...extra }; -} - -export function runValidationCommand( - command: string, - args: readonly string[], - cwd: string, - timeoutMs: number, - extraEnv?: Record, - signal?: AbortSignal, -): Promise { - const displayCommand = [command, ...args].map(quoteArg).join(" "); - const spawnCommand = useTaskpolicy ? TASKPOLICY : command; - const spawnArgs = useTaskpolicy ? ["-c", "background", command, ...args] : [...args]; - const started = Date.now(); - - return new Promise((resolve) => { - let stdout = ""; - let stderr = ""; - let stdoutBytes = 0; - let stderrBytes = 0; - let timedOut = false; - let outputLimited = false; - let settled = false; - let lastActivityAtMs = started; - - const settle = (result: CommandResult) => { - if (settled) return; - settled = true; - clearTimeout(timer); - signal?.removeEventListener("abort", onAbort); - resolve(result); - }; - - const append = (stream: "stdout" | "stderr", chunk: Buffer) => { - lastActivityAtMs = Date.now(); - const text = chunk.toString(); - if (stream === "stdout") { - stdoutBytes += chunk.length; - stdout = (stdout + text).slice(-RETAINED_OUTPUT_CHARS); - } else { - stderrBytes += chunk.length; - stderr = (stderr + text).slice(-RETAINED_OUTPUT_CHARS); - } - if ( - !outputLimited && - (stdoutBytes > VALIDATION_OUTPUT_LIMIT_BYTES || stderrBytes > VALIDATION_OUTPUT_LIMIT_BYTES) - ) { - outputLimited = true; - tree.terminate(); - } - }; - - const tree = spawnProcessTree( - spawnCommand, - spawnArgs, - { cwd, env: validationEnv(extraEnv) }, - { - onStdout: (chunk) => append("stdout", chunk), - onStderr: (chunk) => append("stderr", chunk), - onError: (cause) => { - const code = typeof cause.code === "string" ? cause.code : undefined; - settle({ - command: displayCommand, - exitCode: 127, - timedOut: false, - spawnError: true, - spawnErrorCode: code, - durationMs: Date.now() - started, - stdout: "", - stderr: `${displayCommand}: ${cause.message}`, - stdoutTail: "", - stderrTail: tail(`${displayCommand}: ${cause.message}`), - startedAtMs: started, - lastActivityAtMs, - }); - }, - onClose: (code) => { - tree.kill(); - if (outputLimited) { - stderr += `\n[scaffbench] output exceeded ${VALIDATION_OUTPUT_LIMIT_BYTES} bytes; process tree terminated`; - } - settle({ - command: displayCommand, - exitCode: timedOut ? null : outputLimited ? 1 : code, - timedOut, - timeoutKind: timedOut ? "hard" : undefined, - durationMs: Date.now() - started, - stdout, - stderr, - stdoutTail: tail(stdout), - stderrTail: tail(stderr), - startedAtMs: started, - lastActivityAtMs, - }); - }, - }, - ); - - const onAbort = () => { - tree.terminate(); - }; - - const timer = setTimeout(() => { - timedOut = true; - tree.terminate(); - }, timeoutMs); - timer.unref(); - - if (signal) { - if (signal.aborted) onAbort(); - else signal.addEventListener("abort", onAbort, { once: true }); - } - }); -} diff --git a/scripts/scaffbench/validation/go.ts b/scripts/scaffbench/validation/go.ts deleted file mode 100644 index 9e613aa8d..000000000 --- a/scripts/scaffbench/validation/go.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateGoProject } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/index.ts b/scripts/scaffbench/validation/index.ts deleted file mode 100644 index d374018eb..000000000 --- a/scripts/scaffbench/validation/index.ts +++ /dev/null @@ -1,1499 +0,0 @@ -import type { - BenchmarkSpec, - CommandResult, - ProjectValidation, - RunResult, - ScaffbenchOptions, - StepResult, -} from "@scaffbench/types"; - -import { tail } from "@scaffbench/agents/command"; -import { - bfSpec, - VALIDATION_PROJECT_TIMEOUT_MS, - VALIDATION_ROOT_CAP, - VALIDATION_TIMEOUT_MS, -} from "@scaffbench/constants"; -import { isAdvisoryStep, stepBaseName, typecheckGate } from "@scaffbench/scoring"; -import { runValidationCommand } from "@scaffbench/validation/executor"; -import * as Duration from "effect/Duration"; -import * as Effect from "effect/Effect"; -import * as Option from "effect/Option"; -import { existsSync, readdirSync } from "node:fs"; -import { cp, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; -import path from "node:path"; - -const INSTALL_STEP_KEYS = new Set(["install", "dotnetRestore"]); -import { hasTransientNetworkSignature } from "@scaffbench/validation/classification"; -import { parseJsonc, walk } from "@scaffbench/validation/shared"; - -function fromPromise(evaluate: () => Promise) { - return Effect.tryPromise({ try: evaluate, catch: (cause) => cause }); -} - -function runCommand( - command: string, - args: readonly string[], - cwd: string, - env?: Record, -) { - return Effect.tryPromise({ - try: (signal: AbortSignal) => - runValidationCommand(command, args, cwd, VALIDATION_TIMEOUT_MS, env, signal), - catch: (cause) => cause, - }); -} - -type ValidationSteps = Record; - -type ValidationGate = { - key: string; - nonBlocking?: boolean; - run: () => Effect.Effect; -}; - -function emptySteps(): ValidationSteps { - return {}; -} - -function runGates(gates: readonly ValidationGate[], steps: ValidationSteps = {}) { - return Effect.gen(function* () { - let halted = false; - for (const gate of gates) { - if (halted) { - steps[`not-run:${gate.key}`] = notRunStep( - `${gate.key} not run: an earlier validation step failed`, - ); - continue; - } - const step = yield* gate.run(); - steps[gate.key] = step; - if (!gate.nonBlocking && stepFailed(step)) halted = true; - } - return steps; - }); -} - -type CommandStepOptions = { - retryTransientNetwork?: boolean; - env?: Record; -}; - -export function commandStep( - command: string, - args: readonly string[], - cwd: string, - options: CommandStepOptions = {}, -) { - return Effect.gen(function* () { - const first = toStep(yield* runCommand(command, args, cwd, options.env)); - if ( - !options.retryTransientNetwork || - first.exitCode === 0 || - !hasTransientNetworkSignature(first) - ) { - return first; - } - const retry = toStep(yield* runCommand(command, args, cwd, options.env)); - return { - ...retry, - durationMs: first.durationMs + retry.durationMs, - retryCount: 1, - transientNetwork: - retry.exitCode !== 0 && hasTransientNetworkSignature(retry) ? true : undefined, - }; - }); -} - -const PROJECT_MANIFESTS = [ - "package.json", - "Cargo.toml", - "go.mod", - "pyproject.toml", - "bts.jsonc", - "pom.xml", - "build.gradle", - "build.gradle.kts", - "mix.exs", - "global.json", -]; - -function hasDotnetManifest(dir: string) { - try { - return readdirSync(dir).some( - (name) => name.endsWith(".csproj") || name.endsWith(".sln") || name.endsWith(".slnx"), - ); - } catch { - return false; - } -} - -export async function findProjectDir(runDir: string, projectName: string) { - const expected = path.join(runDir, projectName); - if (existsSync(expected)) return expected; - - const entries = await readdir(runDir, { withFileTypes: true }); - const dirs = entries.filter( - (entry) => entry.isDirectory() && !entry.name.startsWith(".") && entry.name !== "node_modules", - ); - if (dirs.length === 1 && dirs[0]) return path.join(runDir, dirs[0].name); - - const withManifest = dirs.filter( - (dir) => - PROJECT_MANIFESTS.some((manifest) => existsSync(path.join(runDir, dir.name, manifest))) || - hasDotnetManifest(path.join(runDir, dir.name)), - ); - if (withManifest.length === 1 && withManifest[0]) { - return path.join(runDir, withManifest[0].name); - } - return null; -} - -export async function archiveProjectSource(srcDir: string, destDir: string) { - const skip = new Set([ - "node_modules", - ".git", - "dist", - "build", - ".next", - ".turbo", - "coverage", - "target", - ".venv", - "bin", - "obj", - "deps", - "_build", - ]); - await rm(destDir, { recursive: true, force: true }); - await cp(srcDir, destDir, { - recursive: true, - force: true, - filter: (source) => !skip.has(path.basename(source)), - }); -} - -async function findManifestRoots(projectDir: string, manifests: readonly string[]) { - const roots = new Set(); - await walk(projectDir, async (filePath) => { - if (manifests.includes(path.basename(filePath))) roots.add(path.dirname(filePath)); - }); - return [...roots].sort((a, b) => a.length - b.length || a.localeCompare(b)); -} - -function dropNestedRoots(roots: string[]) { - const kept: string[] = []; - for (const root of roots) { - if (!kept.some((k) => root === k || root.startsWith(k + path.sep))) kept.push(root); - } - return kept; -} - -function isNestedRoot(parent: string, candidate: string) { - return candidate !== parent && candidate.startsWith(`${parent}${path.sep}`); -} - -function expandBraces(pattern: string): string[] { - const match = pattern.match(/\{([^{}]+)\}/); - if (!match || match.index === undefined) return [pattern]; - return match[1]! - .split(",") - .flatMap((choice) => - expandBraces( - `${pattern.slice(0, match.index)}${choice}${pattern.slice(match.index! + match[0].length)}`, - ), - ); -} - -function workspaceGlobMatches(pattern: string, relativeRoot: string) { - const normalizedPattern = pattern.replace(/^\.\//, "").replace(/\/$/, ""); - const normalizedRoot = relativeRoot.split(path.sep).join("/").replace(/^\.\//, ""); - let source = ""; - for (let index = 0; index < normalizedPattern.length; index += 1) { - const character = normalizedPattern[index]!; - if (character === "*") { - if (normalizedPattern[index + 1] === "*") { - index += 1; - if (normalizedPattern[index + 1] === "/") { - index += 1; - source += "(?:.*/)?"; - } else { - source += ".*"; - } - } else { - source += "[^/]*"; - } - } else if (character === "?") { - source += "[^/]"; - } else { - source += character.replace(/[|\\{}()[\]^$+?.-]/g, "\\$&"); - } - } - return new RegExp(`^${source}$`).test(normalizedRoot); -} - -function workspacePatternsCover(patterns: readonly string[], relativeRoot: string) { - let covered = false; - for (const rawPattern of patterns) { - const excluded = rawPattern.startsWith("!"); - const pattern = excluded ? rawPattern.slice(1) : rawPattern; - if (expandBraces(pattern).some((candidate) => workspaceGlobMatches(candidate, relativeRoot))) { - covered = !excluded; - } - } - return covered; -} - -async function bunWorkspacePatterns(root: string) { - const packageJson = await readPackageJson(path.join(root, "package.json")); - const workspaces = Array.isArray(packageJson.workspaces) - ? packageJson.workspaces - : packageJson.workspaces?.packages; - return Array.isArray(workspaces) - ? workspaces.filter((entry): entry is string => typeof entry === "string") - : []; -} - -async function cargoWorkspacePatterns(root: string) { - const manifest = await readOptional(path.join(root, "Cargo.toml")); - if (!manifest) return []; - const workspace = manifest.match( - /^\s*\[workspace\]\s*$([\s\S]*?)(?=^\s*\[[^\]]+\]\s*$|(?![\s\S]))/m, - )?.[1]; - if (!workspace) return []; - const values = (field: "members" | "exclude") => { - const body = workspace.match(new RegExp(`\\b${field}\\s*=\\s*\\[([\\s\\S]*?)\\]`))?.[1] ?? ""; - return [...body.matchAll(/["']([^"']+)["']/g)].map((match) => match[1]!); - }; - return [...values("members"), ...values("exclude").map((entry) => `!${entry}`)]; -} - -const MANIFEST_FIELDS = ["name", "scripts", "dependencies", "devDependencies", "workspaces"]; - -async function isRealBunManifest(root: string) { - const packageJson = await readPackageJson(path.join(root, "package.json")); - return MANIFEST_FIELDS.some((field) => packageJson[field] !== undefined); -} - -async function findBunManifestRoots(projectDir: string) { - const roots = await findManifestRoots(projectDir, ["package.json"]); - const real = await Promise.all(roots.map(isRealBunManifest)); - return roots.filter((_, index) => real[index]); -} - -async function membershipAwareRoots( - roots: readonly string[], - patternsFor: (root: string) => Promise, -) { - const patterns = new Map(); - for (const root of roots) patterns.set(root, await patternsFor(root)); - return roots.filter( - (candidate) => - !roots.some((parent) => { - if (!isNestedRoot(parent, candidate)) return false; - return workspacePatternsCover(patterns.get(parent) ?? [], path.relative(parent, candidate)); - }), - ); -} - -async function coveredWorkspaceMembers( - parent: string, - roots: readonly string[], - patternsFor: (root: string) => Promise, -) { - const patterns = await patternsFor(parent); - return roots.filter( - (candidate) => - isNestedRoot(parent, candidate) && - workspacePatternsCover(patterns, path.relative(parent, candidate)), - ); -} - -export type ValidationLimits = { - deadlineMs?: number; - rootCap?: number; -}; - -export function effectiveValidationOptions( - spec: BenchmarkSpec, - options: ScaffbenchOptions, -): ScaffbenchOptions { - const profile = spec.validationProfile; - return { - ...options, - qualityGate: - options.noQualityGate === true ? false : options.qualityGate || profile.qualityGate === true, - doctorCheck: options.doctorCheck && profile.doctorCheck === true, - routeCheck: options.routeCheck && profile.routeCheckCandidate === true, - }; -} - -export function validateProject( - spec: BenchmarkSpec, - projectDir: string | null, - requestedOptions: ScaffbenchOptions, - limits: ValidationLimits = {}, -) { - const options = effectiveValidationOptions(spec, requestedOptions); - const qualityGateRequested = options.qualityGate; - if (!projectDir) { - return Effect.succeed({ - projectExists: false, - qualityGateRequested, - steps: {}, - } as ProjectValidation); - } - const steps: Record = {}; - const rootCap = limits.rootCap ?? VALIDATION_ROOT_CAP; - let rootsUsed = 0; - - const validation = Effect.gen(function* () { - const prefixFor = (root: string) => - root === projectDir ? "" : path.relative(projectDir, root).split(path.sep).join("/"); - const merge = ( - incoming: Record, - prefix: string, - eco: string, - ) => { - let merged = 0; - for (const [key, step] of Object.entries(incoming)) { - if (!step) continue; - const target = prefix ? `${prefix}:${key}` : key; - steps[steps[target] === undefined ? target : `${eco}:${key}`] = step; - merged += 1; - } - if (merged === 0) { - steps[`unvalidated:${eco}:${prefix || "."}`] = unvalidatedStep( - `${eco} manifest discovered at ${prefix || "."} but no validator step ran`, - ); - } - }; - const takeRoots = (roots: readonly string[], eco: string) => { - const accepted: string[] = []; - for (const root of roots) { - if (rootsUsed < rootCap) { - rootsUsed += 1; - accepted.push(root); - continue; - } - const relative = prefixFor(root) || "."; - steps[`unvalidated:${eco}:${relative}`] = unvalidatedStep( - `${eco} root ${relative} exceeded validation root cap ${rootCap}`, - ); - } - return accepted; - }; - const subOptions = { ...options, doctorCheck: false, routeCheck: false }; - const bun = existsSync(`${process.env.HOME}/.bun/bin/bun`) - ? `${process.env.HOME}/.bun/bin/bun` - : "bun"; - const installedRoots = new Set(); - const prerequisiteEnv = (root: string) => ({ - PATH: [ - path.join(root, "node_modules", ".bin"), - path.join(projectDir, "node_modules", ".bin"), - process.env.PATH ?? "", - ].join(path.delimiter), - }); - - for (const [index, prerequisite] of (spec.prerequisiteCommands ?? []).entries()) { - const [command, ...args] = prerequisite.command; - const label = `prerequisite:${String(index + 1).padStart(2, "0")}:${command ?? "missing"}`; - if (!command) { - steps[label] = skipStep("empty prerequisite command"); - return buildProjectValidation(steps, qualityGateRequested); - } - const configs = prerequisite.whenConfigFound; - const roots = configs - ? yield* fromPromise(() => findManifestRoots(projectDir, configs)) - : [projectDir]; - if (roots.length === 0) { - steps[label] = naStep(`${command} (no ${configs?.join(" / ")} in the project)`); - continue; - } - for (const root of roots) { - const key = root === projectDir ? label : `${label}:${prefixFor(root)}`; - const installRoot = [root, projectDir].find((dir) => - existsSync(path.join(dir, "package.json")), - ); - if (installRoot && !installedRoots.has(installRoot)) { - installedRoots.add(installRoot); - const installKey = `${key}:install`; - steps[installKey] = yield* commandStep( - bun, - ["install", "--concurrent-scripts=2", "--network-concurrency=8"], - installRoot, - { retryTransientNetwork: true }, - ); - if (!stepGreen(steps[installKey])) { - return buildProjectValidation(steps, qualityGateRequested); - } - } - steps[key] = yield* commandStep(command, args, root, { env: prerequisiteEnv(root) }); - if (!stepGreen(steps[key])) { - return buildProjectValidation(steps, qualityGateRequested); - } - } - } - const prerequisiteStepCount = Object.keys(steps).length; - - const coreVerdictFailed = () => - Object.entries(steps).some( - ([name, step]) => - !name.startsWith("unvalidated:") && !isAdvisoryStep(name) && stepFailed(step), - ); - const recordNotRun = (eco: string, root: string) => { - steps[`not-run:${eco}:${prefixFor(root) || "."}`] = notRunStep( - `${eco} validation of ${prefixFor(root) || "."} not run: an earlier core step already failed`, - ); - }; - - const allBunRoots = yield* fromPromise(() => findBunManifestRoots(projectDir)); - const bunRoots = yield* fromPromise(() => - membershipAwareRoots(allBunRoots, bunWorkspacePatterns), - ); - for (const root of takeRoots(bunRoots, "bun")) { - if (coreVerdictFailed()) { - recordNotRun("bun", root); - continue; - } - const isRoot = root === projectDir; - const bunSteps = yield* validateBunProject(root, isRoot ? options : subOptions); - merge(bunSteps, prefixFor(root), "bun"); - if (isRoot && bunSteps.install?.exitCode === 0 && !bunSteps.build && !bunSteps.typecheck) { - const members = yield* fromPromise(() => - coveredWorkspaceMembers(root, allBunRoots, bunWorkspacePatterns), - ); - for (const member of takeRoots(members, "bun")) { - if (coreVerdictFailed()) { - recordNotRun("bun", member); - continue; - } - merge(yield* validateBunProject(member, subOptions), prefixFor(member), "bun"); - } - } - } - - const nativeProfiles = new Set(spec.validationProfile.native ?? []); - const allCargoRoots = yield* fromPromise(() => findManifestRoots(projectDir, ["Cargo.toml"])); - const cargoRoots = yield* fromPromise(() => - membershipAwareRoots(allCargoRoots, cargoWorkspacePatterns), - ); - for (const root of takeRoots(cargoRoots, "cargo")) { - if (coreVerdictFailed()) { - recordNotRun("cargo", root); - continue; - } - merge( - yield* validateCargoProject(root, root === projectDir ? options : subOptions), - prefixFor(root), - "cargo", - ); - } - const pythonRoots = dropNestedRoots( - yield* fromPromise(() => findManifestRoots(projectDir, ["pyproject.toml"])), - ); - for (const root of takeRoots(pythonRoots, "python")) { - if (coreVerdictFailed()) { - recordNotRun("python", root); - continue; - } - merge( - yield* validatePythonProject(root, root === projectDir ? options : subOptions), - prefixFor(root), - "python", - ); - } - const requirementsRoots = dropNestedRoots( - yield* fromPromise(() => findManifestRoots(projectDir, ["requirements.txt"])), - ).filter( - (root) => !pythonRoots.some((python) => root === python || isNestedRoot(python, root)), - ); - for (const root of takeRoots(requirementsRoots, "python")) { - if (coreVerdictFailed()) { - recordNotRun("python", root); - continue; - } - merge( - yield* validatePythonRequirementsProject(root, root === projectDir ? options : subOptions), - prefixFor(root), - "python", - ); - } - const goRoots = yield* fromPromise(() => findManifestRoots(projectDir, ["go.mod"])); - for (const root of takeRoots(goRoots, "go")) { - if (coreVerdictFailed()) { - recordNotRun("go", root); - continue; - } - merge( - yield* validateGoProject(root, root === projectDir ? options : subOptions), - prefixFor(root), - "go", - ); - } - if (nativeProfiles.has("dotnet") || (yield* fromPromise(() => hasDotnetProject(projectDir)))) { - if (coreVerdictFailed()) { - recordNotRun("dotnet", projectDir); - } else { - merge( - yield* validateDotnetProject(projectDir, options, { - targetCap: Math.max(0, rootCap - rootsUsed), - }), - "", - "dotnet", - ); - } - } - if (nativeProfiles.has("java")) { - if (coreVerdictFailed()) { - recordNotRun("java", projectDir); - } else { - merge(yield* validateJavaProject(projectDir, options), "", "java"); - } - } - if (nativeProfiles.has("elixir")) { - if (coreVerdictFailed()) { - recordNotRun("elixir", projectDir); - } else { - merge(yield* validateElixirProject(projectDir, options), "", "elixir"); - } - } - - if (Object.keys(steps).length === prerequisiteStepCount) { - steps["unvalidated:project"] = unvalidatedStep( - "project directory has no recognized build manifest", - ); - } else { - const substantiveCore = Object.entries(steps).some( - ([name, step]) => - step !== undefined && - step.status !== "skip" && - step.status !== "na" && - !isAdvisoryStep(name) && - !name.startsWith("prerequisite:") && - !INSTALL_STEP_KEYS.has(stepBaseName(name)), - ); - if (!substantiveCore) { - steps["unvalidated:no-build-surface"] = unvalidatedStep( - "no build or typecheck surface was discovered. A green install alone is not a pass", - ); - } - } - return buildProjectValidation(steps, qualityGateRequested); - }); - - return validation.pipe( - Effect.timeoutOption(Duration.millis(limits.deadlineMs ?? VALIDATION_PROJECT_TIMEOUT_MS)), - Effect.map( - Option.getOrElse(() => { - steps["unvalidated:deadline"] = unvalidatedStep( - `project validation exceeded ${limits.deadlineMs ?? VALIDATION_PROJECT_TIMEOUT_MS}ms deadline`, - ); - return buildProjectValidation(steps, qualityGateRequested); - }), - ), - ); -} - -export function validateBunProject(projectDir: string, options: ScaffbenchOptions) { - return Effect.gen(function* () { - const packageJsonPath = path.join(projectDir, "package.json"); - if (!existsSync(packageJsonPath)) return emptySteps(); - - const bun = existsSync(`${process.env.HOME}/.bun/bin/bun`) - ? `${process.env.HOME}/.bun/bin/bun` - : "bun"; - const bunx = existsSync(`${process.env.HOME}/.bun/bin/bunx`) - ? `${process.env.HOME}/.bun/bin/bunx` - : "bunx"; - const packageJson = yield* fromPromise(() => readPackageJson(packageJsonPath)); - const scripts = (packageJson.scripts ?? {}) as Record; - - const gates: ValidationGate[] = [ - { - key: "install", - run: () => - commandStep( - bun, - ["install", "--concurrent-scripts=2", "--network-concurrency=8"], - projectDir, - { retryTransientNetwork: true }, - ), - }, - ]; - - const expoCommand = expoExportCommand(packageJson); - if (expoCommand) { - gates.push({ - key: "build", - run: () => - commandStep(expoCommand.command, expoCommand.args, projectDir, { - env: { CI: "1", EXPO_NO_TELEMETRY: "1" }, - }), - }); - } else if (scripts.build) { - gates.push({ key: "build", run: () => commandStep(bun, ["run", "build"], projectDir) }); - } - - const gate = typecheckGate(scripts, existsSync(path.join(projectDir, "tsconfig.json"))); - if (gate === "tsc") { - gates.push({ - key: "typecheck", - run: () => commandStep(bunx, ["tsc", "--build"], projectDir), - }); - } else if (gate) { - gates.push({ key: "typecheck", run: () => commandStep(bun, ["run", gate], projectDir) }); - } - - if (options.qualityGate || scripts.lint) { - gates.push({ key: "lint", run: () => bunLintStep(projectDir, bun, scripts) }); - } - if (options.qualityGate) { - gates.push( - { key: "format", run: () => bunFormatStep(projectDir, bun, scripts) }, - { - key: "test", - nonBlocking: true, - run: () => - scripts.test - ? commandStep(bun, ["run", "test"], projectDir) - : Effect.succeed(naStep("test (no test script)")), - }, - ); - } - if (options.doctorCheck) { - gates.push({ - key: "doctor", - nonBlocking: true, - run: () => - existsSync(path.join(projectDir, "bts.jsonc")) - ? commandStep( - bunx, - [bfSpec("create-better-fullstack"), "doctor", ".", "--skip-checks", "--json"], - projectDir, - ) - : Effect.succeed(naStep("doctor (not a Better-Fullstack project)")), - }); - } - if (options.routeCheck) { - gates.push({ - key: "route", - nonBlocking: true, - run: () => - scripts.dev - ? fromPromise(() => runProjectRouteCheck(projectDir, options.outDir)) - : Effect.succeed(naStep("route-check (no dev script)")), - }); - } - - return yield* runGates(gates); - }); -} - -function bunLintStep(projectDir: string, bun: string, scripts: Record) { - if (scripts.lint) return commandStep(bun, ["run", "lint"], projectDir); - const biomeBin = localBin(projectDir, "biome"); - if (biomeBin) return commandStep(biomeBin, ["lint", "."], projectDir); - const eslintBin = localBin(projectDir, "eslint"); - if (eslintBin) return commandStep(eslintBin, ["."], projectDir); - return Effect.succeed(skipStep("lint (no linter configured)")); -} - -function bunFormatStep(projectDir: string, bun: string, scripts: Record) { - const formatCheck = formatCheckCommand(scripts, projectDir, bun); - return formatCheck - ? commandStep(formatCheck.command, formatCheck.args, projectDir) - : Effect.succeed(skipStep("format (no formatter configured)")); -} - -async function runProjectRouteCheck(projectDir: string, outDir: string): Promise { - const config = await readRouteCheckConfig(projectDir); - if (!config) return naStep("route-check (missing Better-Fullstack route metadata)"); - - const start = Date.now(); - let handle: any = null; - try { - const devCheck = await import("@testing/lib/dev-check"); - const routeCheck = await import("@testing/lib/route-check"); - handle = await devCheck.startDevServer(projectDir, config); - const result = await routeCheck.runRouteCheck( - handle, - path.join(outDir, "route-check", path.basename(projectDir)), - ); - return verifyStepToHarnessStep(result); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - return { - command: "route-check", - exitCode: 1, - timedOut: false, - durationMs: Date.now() - start, - stdoutTail: tail(handle?.stdoutBuf?.() ?? ""), - stderrTail: tail(`${message}\n${handle?.stderrBuf?.() ?? ""}`), - }; - } finally { - if (handle) { - try { - const devCheck = await import("@testing/lib/dev-check"); - await devCheck.stopDevServer(handle); - } catch {} - } - } -} - -async function readRouteCheckConfig(projectDir: string) { - const btsPath = path.join(projectDir, "bts.jsonc"); - if (!existsSync(btsPath)) return null; - - const parsed = parseJsonc(await readFile(btsPath, "utf8")); - if (!parsed) return null; - - const frontend = inferFrontend(parsed); - if (frontend.every((entry) => entry === "none")) return null; - - return { - ...parsed, - projectName: parsed.projectName ?? path.basename(projectDir), - projectDir, - relativePath: parsed.relativePath ?? ".", - frontend, - }; -} - -function inferFrontend(config: Record): string[] { - if (Array.isArray(config.frontend)) return config.frontend.filter(Boolean); - if (typeof config.frontend === "string" && config.frontend) return [config.frontend]; - - if (Array.isArray(config.stackParts)) { - const frontendPart = config.stackParts.find( - (part: Record) => part.role === "frontend" && typeof part.toolId === "string", - ); - if (frontendPart?.toolId) return [frontendPart.toolId]; - } - - return []; -} - -function verifyStepToHarnessStep(result: any): StepResult { - return { - command: result.step ?? "route-check", - exitCode: result.success || result.skipped ? 0 : (result.exitCode ?? 1), - timedOut: Boolean(result.timedOut), - durationMs: result.durationMs ?? 0, - stdoutTail: tail(result.stdout ?? ""), - stderrTail: tail(result.stderr ?? ""), - }; -} - -export function validateCargoProject(projectDir: string, options: ScaffbenchOptions) { - return Effect.gen(function* () { - if (!existsSync(path.join(projectDir, "Cargo.toml"))) return emptySteps(); - const gates: ValidationGate[] = [ - { - key: "cargoCheck", - run: () => commandStep("cargo", ["check", "--workspace", "--all-targets"], projectDir), - }, - ]; - if (options.qualityGate) { - gates.push( - { key: "format", run: () => commandStep("cargo", ["fmt", "--check"], projectDir) }, - { - key: "lint", - run: () => commandStep("cargo", ["clippy", "--", "-D", "warnings"], projectDir), - }, - { key: "test", run: () => commandStep("cargo", ["test"], projectDir) }, - ); - } - return yield* runGates(gates); - }); -} - -export function validatePythonProject(projectDir: string, options: ScaffbenchOptions) { - return Effect.gen(function* () { - if (!existsSync(path.join(projectDir, "pyproject.toml"))) return emptySteps(); - const gates: ValidationGate[] = [ - { - key: "install", - run: () => - commandStep("uv", ["sync", "--all-extras"], projectDir, { retryTransientNetwork: true }), - }, - { - key: "compile", - run: () => - commandStep( - "uv", - ["run", "python", "-m", "compileall", "-q", pythonSourceDir(projectDir)], - projectDir, - ), - }, - { key: "typecheck", run: () => pythonTypecheckStep(projectDir) }, - ]; - if (options.qualityGate) { - gates.push( - { key: "lint", run: () => commandStep("uv", ["run", "ruff", "check", "."], projectDir) }, - { - key: "format", - run: () => commandStep("uv", ["run", "ruff", "format", "--check", "."], projectDir), - }, - { key: "test", run: () => pytestStep(commandStep("uv", ["run", "pytest"], projectDir)) }, - ); - } - return yield* runGates(gates); - }); -} - -export function validatePythonRequirementsProject(projectDir: string, options: ScaffbenchOptions) { - return Effect.gen(function* () { - if (!existsSync(path.join(projectDir, "requirements.txt"))) return emptySteps(); - const python = path.join(projectDir, ".venv", "bin", "python"); - const ruff = path.join(projectDir, ".venv", "bin", "ruff"); - const pytestBin = path.join(projectDir, ".venv", "bin", "pytest"); - const gates: ValidationGate[] = [ - { key: "install", run: () => pipInstallStep(projectDir) }, - { - key: "compile", - run: () => - commandStep( - python, - ["-m", "compileall", "-q", "-x", "[\\\\/]\\.venv[\\\\/]", pythonSourceDir(projectDir)], - projectDir, - ), - }, - { key: "typecheck", run: () => pythonImportSmokeStep(projectDir, python, []) }, - ]; - if (options.qualityGate) { - gates.push( - { - key: "lint", - run: () => - existsSync(ruff) - ? commandStep(ruff, ["check", "."], projectDir) - : Effect.succeed(skipStep("lint (no linter installed)")), - }, - { - key: "format", - run: () => - existsSync(ruff) - ? commandStep(ruff, ["format", "--check", "."], projectDir) - : Effect.succeed(skipStep("format (no formatter installed)")), - }, - { - key: "test", - run: () => - existsSync(pytestBin) - ? pytestStep(commandStep(pytestBin, [], projectDir)) - : Effect.succeed(naStep("pytest (not installed)")), - }, - ); - } - return yield* runGates(gates); - }); -} - -function pythonSourceDir(projectDir: string) { - return existsSync(path.join(projectDir, "src")) ? "src/" : "."; -} - -function pipInstallStep(projectDir: string) { - return Effect.gen(function* () { - const venv = yield* commandStep("uv", ["venv"], projectDir); - if (!stepGreen(venv)) return venv; - return yield* commandStep("uv", ["pip", "install", "-r", "requirements.txt"], projectDir, { - retryTransientNetwork: true, - }); - }); -} - -function pythonTypecheckStep(projectDir: string) { - return Effect.gen(function* () { - const typechecker = yield* fromPromise(() => configuredPythonTypechecker(projectDir)); - if (typechecker === "mypy") { - const mypyArguments = (yield* fromPromise(() => pythonMypyHasConfiguredTargets(projectDir))) - ? [] - : ["."]; - return yield* commandStep("uv", ["run", "mypy", ...mypyArguments], projectDir); - } - if (typechecker === "pyright") { - return yield* commandStep("uv", ["run", "pyright"], projectDir); - } - return yield* pythonImportSmokeStep(projectDir, "uv", ["run", "python"]); - }); -} - -function pythonImportSmokeStep( - projectDir: string, - command: string, - argumentPrefix: readonly string[], -) { - return Effect.gen(function* () { - const entryTarget = yield* fromPromise(() => findPythonEntryTarget(projectDir)); - if (!entryTarget) return skipStep("python import smoke (no importable package found)"); - return yield* commandStep( - command, - [...argumentPrefix, "-c", pythonFileImportCommand(entryTarget)], - projectDir, - ); - }); -} - -function pytestStep(run: Effect.Effect) { - return Effect.map(run, (pytest) => - pytest.exitCode === 5 ? naStep("pytest (no tests collected)") : pytest, - ); -} - -export function validateGoProject(projectDir: string, options: ScaffbenchOptions) { - return Effect.gen(function* () { - if (!existsSync(path.join(projectDir, "go.mod"))) return emptySteps(); - const gates: ValidationGate[] = [ - { - key: "install", - run: () => - commandStep("go", ["mod", "download"], projectDir, { retryTransientNetwork: true }), - }, - { key: "build", run: () => commandStep("go", ["build", "./..."], projectDir) }, - { key: "tidy", nonBlocking: true, run: () => runGoTidyAdvisory(projectDir) }, - ]; - if (options.qualityGate) { - gates.push( - { key: "lint", run: () => commandStep("go", ["vet", "./..."], projectDir) }, - { key: "format", run: () => gofmtStep(projectDir) }, - { key: "test", run: () => commandStep("go", ["test", "./..."], projectDir) }, - ); - } - return yield* runGates(gates); - }); -} - -function gofmtStep(projectDir: string) { - return Effect.gen(function* () { - const gofmt = yield* runCommand("gofmt", ["-l", "."], projectDir); - const unformatted = gofmt.stdout.trim(); - return toStep( - gofmt.exitCode === 0 && unformatted - ? { - ...gofmt, - exitCode: 1, - stderr: `gofmt: ${unformatted.split("\n").filter(Boolean).length} file(s) need formatting:\n${unformatted}`, - } - : gofmt, - ); - }); -} - -export function validateDotnetProject( - projectDir: string, - options: ScaffbenchOptions, - limits: { targetCap?: number } = {}, -) { - return Effect.gen(function* () { - const steps: Record = {}; - const targets = yield* fromPromise(() => dotnetValidationTargets(projectDir)); - const targetCap = limits.targetCap ?? Number.POSITIVE_INFINITY; - let targetsUsed = 0; - const acceptTarget = (target: string) => { - if (targetsUsed < targetCap) { - targetsUsed += 1; - return true; - } - const namespace = target.endsWith(".csproj") - ? dotnetStepNamespace(projectDir, target) - : path.relative(projectDir, target).split(path.sep).join("/"); - steps[`unvalidated:dotnet:${namespace}`] = unvalidatedStep( - `dotnet target ${namespace} exceeded validation root cap ${targetCap}`, - ); - return false; - }; - if (targets.kind === "solution") { - const solution = targets.targets[0]!; - if (acceptTarget(solution)) { - const root = path.dirname(solution); - const target = path.basename(solution); - yield* runGates(dotnetGates(root, target, "", options), steps); - } - } - - const dotnetCoreFailed = () => - Object.entries(steps).some( - ([name, step]) => - !name.startsWith("unvalidated:") && !isAdvisoryStep(name) && stepFailed(step), - ); - const projects = targets.kind === "solution" ? targets.uncoveredProjects : targets.targets; - for (const project of projects) { - if (!acceptTarget(project)) continue; - const root = path.dirname(project); - const target = path.basename(project); - const namespace = dotnetStepNamespace(projectDir, project); - if (dotnetCoreFailed()) { - steps[`${namespace}:not-run`] = notRunStep( - `dotnet target ${namespace} not run: an earlier core step already failed`, - ); - continue; - } - yield* runGates(dotnetGates(root, target, `${namespace}:`, options), steps); - } - return steps; - }); -} - -function dotnetGates( - root: string, - target: string, - keyPrefix: string, - options: ScaffbenchOptions, -): ValidationGate[] { - const gates: ValidationGate[] = [ - { - key: `${keyPrefix}dotnetRestore`, - run: () => commandStep("dotnet", ["restore", target], root, { retryTransientNetwork: true }), - }, - { - key: `${keyPrefix}dotnetBuild`, - run: () => commandStep("dotnet", ["build", target, "--no-restore"], root), - }, - ]; - if (options.qualityGate) { - gates.push({ - key: `${keyPrefix}test`, - run: () => commandStep("dotnet", ["test", target, "--no-build"], root), - }); - } - return gates; -} - -export async function findBuildRoot( - projectDir: string, - manifests: readonly string[], -): Promise { - const roots = new Set(); - await walk(projectDir, async (filePath) => { - if (manifests.includes(path.basename(filePath))) roots.add(path.dirname(filePath)); - }); - if (roots.size === 0) return null; - const list = [...roots]; - return ( - list.find((root) => root.endsWith(path.join("apps", "server"))) ?? - list.sort((a, b) => a.length - b.length)[0] ?? - null - ); -} - -export function validateJavaProject(projectDir: string, options: ScaffbenchOptions) { - return Effect.gen(function* () { - const root = yield* fromPromise(() => - findBuildRoot(projectDir, ["pom.xml", "build.gradle", "build.gradle.kts"]), - ); - if (!root) return emptySteps(); - const hasPom = existsSync(path.join(root, "pom.xml")); - const wrapper = hasPom ? "mvnw" : "gradlew"; - const usesWrapper = existsSync(path.join(root, wrapper)); - const [bin, buildArgs, testArgs] = hasPom - ? ([ - usesWrapper ? "./mvnw" : "mvn", - ["-q", "-B", "-DskipTests", "compile"], - ["-q", "-B", "test"], - ] as const) - : ([ - usesWrapper ? "./gradlew" : "gradle", - ["compileJava", "-x", "test", "--console=plain"], - ["test", "--console=plain"], - ] as const); - const gates: ValidationGate[] = [ - { key: "build", run: () => commandStep(bin, [...buildArgs], root) }, - ]; - if (options.qualityGate) { - gates.push({ key: "test", run: () => commandStep(bin, [...testArgs], root) }); - } - return yield* runGates(gates); - }); -} - -export function validateElixirProject(projectDir: string, options: ScaffbenchOptions) { - return Effect.gen(function* () { - const root = yield* fromPromise(() => findBuildRoot(projectDir, ["mix.exs"])); - if (!root) return emptySteps(); - const gates: ValidationGate[] = [ - { - key: "install", - run: () => commandStep("mix", ["deps.get"], root, { retryTransientNetwork: true }), - }, - { key: "build", run: () => commandStep("mix", ["compile"], root) }, - ]; - if (options.qualityGate) { - gates.push( - { key: "format", run: () => commandStep("mix", ["format", "--check-formatted"], root) }, - { key: "test", run: () => commandStep("mix", ["test"], root) }, - ); - } - return yield* runGates(gates); - }); -} - -async function hasDotnetProject(projectDir: string) { - return ( - (await findDotnetProjects(projectDir)).length > 0 || - (await findDotnetSolutions(projectDir)).length > 0 - ); -} - -export async function findDotnetRoots(projectDir: string) { - return [...new Set((await findDotnetProjects(projectDir)).map(path.dirname))]; -} - -export async function findDotnetProjects(projectDir: string) { - const projects: string[] = []; - await walk(projectDir, async (filePath) => { - if (filePath.endsWith(".csproj")) projects.push(filePath); - }); - return projects.sort((a, b) => a.localeCompare(b)); -} - -export async function findDotnetSolutions(projectDir: string) { - const solutions: string[] = []; - await walk(projectDir, async (filePath) => { - if (filePath.endsWith(".sln") || filePath.endsWith(".slnx")) solutions.push(filePath); - }); - return solutions.sort( - (a, b) => - path.relative(projectDir, a).split(path.sep).length - - path.relative(projectDir, b).split(path.sep).length || a.localeCompare(b), - ); -} - -export async function dotnetValidationTargets(projectDir: string) { - const solutions = await findDotnetSolutions(projectDir); - if (solutions.length > 0) { - const solution = solutions[0]!; - const covered = await dotnetSolutionProjects(solution); - const uncoveredProjects = (await findDotnetProjects(projectDir)).filter( - (project) => !covered.has(path.resolve(project)), - ); - return { kind: "solution" as const, targets: [solution], uncoveredProjects }; - } - return { kind: "projects" as const, targets: await findDotnetProjects(projectDir) }; -} - -async function dotnetSolutionProjects(solution: string) { - const text = await readOptional(solution); - const projects = new Set(); - if (!text) return projects; - const root = path.dirname(solution); - const references = solution.endsWith(".slnx") - ? [...text.matchAll(/]*\bPath=["']([^"']+\.csproj)["']/gi)].map( - (match) => match[1]!, - ) - : [...text.matchAll(/Project\([^)]*\)\s*=\s*"[^"]*"\s*,\s*"([^"]+\.csproj)"/gi)].map( - (match) => match[1]!, - ); - for (const reference of references) { - projects.add(path.resolve(root, reference.replace(/[\\/]+/g, path.sep))); - } - return projects; -} - -function dotnetStepNamespace(projectDir: string, project: string) { - const relative = path.relative(projectDir, project).split(path.sep).join("/"); - return relative.replace(/\.csproj$/i, ""); -} - -function toStep(result: CommandResult): StepResult { - const { stdout: _stdout, stderr: _stderr, ...step } = result; - return step; -} - -function stepGreen(step: StepResult | undefined) { - return Boolean( - step && step.status !== "skip" && step.exitCode === 0 && !step.timedOut && !step.spawnError, - ); -} - -function buildProjectValidation( - steps: Record, - qualityGateRequested: boolean, -): ProjectValidation { - const firstByBase = (...names: string[]) => - Object.entries(steps).find( - ([key, step]) => step && names.includes(key.slice(key.lastIndexOf(":") + 1)), - )?.[1]; - return { - projectExists: true, - qualityGateRequested, - steps, - install: steps.install ?? firstByBase("install", "dotnetRestore"), - build: steps.build ?? firstByBase("build", "dotnetBuild", "cargoCheck", "compile"), - checkTypes: steps.typecheck ?? firstByBase("typecheck"), - lint: steps.lint ?? firstByBase("lint"), - format: steps.format ?? firstByBase("format"), - test: steps.test ?? firstByBase("test"), - doctor: steps.doctor, - route: steps.route, - }; -} - -function skipStep(command: string): StepResult { - return { - command, - exitCode: null, - timedOut: false, - status: "skip", - durationMs: 0, - stdoutTail: "skipped (tool not configured)", - stderrTail: "", - }; -} - -function notRunStep(reason: string): StepResult { - return { - command: reason, - exitCode: null, - timedOut: false, - status: "skip", - durationMs: 0, - stdoutTail: "not run (verdict already determined)", - stderrTail: "", - }; -} - -function stepFailed(step: StepResult | undefined) { - return Boolean( - step && - step.status !== "skip" && - step.status !== "na" && - (step.timedOut || step.spawnError === true || (step.exitCode !== null && step.exitCode !== 0)), - ); -} - -function unvalidatedStep(command: string): StepResult { - return { - command, - exitCode: 1, - timedOut: false, - status: "ran", - durationMs: 0, - stdoutTail: "", - stderrTail: command, - }; -} - -function naStep(command: string): StepResult { - return { - command, - exitCode: null, - timedOut: false, - status: "na", - durationMs: 0, - stdoutTail: "n/a", - stderrTail: "", - }; -} - -const FORMAT_CHECK_SCRIPTS = ["format:check", "format-check", "check-format", "fmt:check"]; -const FORMAT_SCRIPT_CANDIDATES = [...FORMAT_CHECK_SCRIPTS, "format", "fmt"]; -const FORMAT_CHECK_FLAG = /(?:^|\s)(?:--check|--check-formatted|--list-different|-l)(?:\s|$)/; -const FORMAT_WRITE_FLAG = /(?:^|\s)(?:--write|--fix|-w)(?:\s|$)/; -const APPENDABLE_CHECK_FORMATTER = /(?:^|\s)(?:vp|oxfmt)(?:\s|$)/; - -export function formatCheckCommand( - scripts: Record, - projectDir: string, - bun: string, -): { command: string; args: readonly string[] } | null { - for (const name of FORMAT_SCRIPT_CANDIDATES) { - const script = scripts[name]; - if (!script || FORMAT_WRITE_FLAG.test(script)) continue; - if (FORMAT_CHECK_FLAG.test(script) || FORMAT_CHECK_SCRIPTS.includes(name)) { - return { command: bun, args: ["run", name] }; - } - if (APPENDABLE_CHECK_FORMATTER.test(script)) { - return { command: bun, args: ["run", name, "--check"] }; - } - } - - const biomeBin = localBin(projectDir, "biome"); - if (biomeBin) return { command: biomeBin, args: ["format", "."] }; - const prettierBin = localBin(projectDir, "prettier"); - if (prettierBin) return { command: prettierBin, args: ["--check", "."] }; - return null; -} - -function localBin(projectDir: string, name: string): string | null { - const p = path.join(projectDir, "node_modules", ".bin", name); - return existsSync(p) ? p : null; -} - -async function readPackageScripts(packageJsonPath: string) { - return ((await readPackageJson(packageJsonPath)).scripts ?? {}) as Record; -} - -async function readPackageJson(packageJsonPath: string): Promise> { - try { - return JSON.parse(await readFile(packageJsonPath, "utf8")); - } catch { - return {}; - } -} - -export function isExpoPackage(packageJson: Record) { - const dependencies = { ...packageJson.dependencies, ...packageJson.devDependencies }; - return typeof dependencies.expo === "string"; -} - -export function expoWebConfigured(packageJson: Record, _projectDir?: string) { - const dependencies = { ...packageJson.dependencies, ...packageJson.devDependencies }; - return Boolean(dependencies["react-dom"] || dependencies["react-native-web"]); -} - -export function expoExportCommand(packageJson: Record) { - if (!isExpoPackage(packageJson)) return null; - return { - command: "npx", - args: expoWebConfigured(packageJson) - ? ["expo", "export", "--platform", "web"] - : ["expo", "export"], - env: { CI: "1", EXPO_NO_TELEMETRY: "1" }, - } as const; -} - -export async function configuredPythonTypechecker( - projectDir: string, -): Promise<"mypy" | "pyright" | null> { - if ( - existsSync(path.join(projectDir, "mypy.ini")) || - existsSync(path.join(projectDir, ".mypy.ini")) - ) { - return "mypy"; - } - if (existsSync(path.join(projectDir, "pyrightconfig.json"))) return "pyright"; - const pyprojectPath = path.join(projectDir, "pyproject.toml"); - const pyproject = existsSync(pyprojectPath) ? await readFile(pyprojectPath, "utf8") : ""; - if (/^\s*\[tool\.mypy\]\s*$/m.test(pyproject)) return "mypy"; - if (/^\s*\[tool\.pyright\]\s*$/m.test(pyproject)) return "pyright"; - return null; -} - -export async function pythonMypyHasConfiguredTargets(projectDir: string): Promise { - const targetPattern = /^\s*(files|packages|modules)\s*=/m; - for (const name of ["mypy.ini", ".mypy.ini"]) { - const iniPath = path.join(projectDir, name); - if (existsSync(iniPath) && targetPattern.test(await readFile(iniPath, "utf8"))) return true; - } - const pyprojectPath = path.join(projectDir, "pyproject.toml"); - if (!existsSync(pyprojectPath)) return false; - const pyproject = await readFile(pyprojectPath, "utf8"); - const headerMatch = /^\s*\[tool\.mypy\]\s*$/m.exec(pyproject); - if (!headerMatch) return false; - const afterHeader = pyproject.slice(headerMatch.index + headerMatch[0].length); - const nextTable = afterHeader.search(/^\s*\[/m); - const section = nextTable === -1 ? afterHeader : afterHeader.slice(0, nextTable); - return targetPattern.test(section); -} - -export async function findPythonEntryModule(projectDir: string): Promise { - return (await findPythonEntryTarget(projectDir))?.moduleName ?? null; -} - -type PythonEntryTarget = { - moduleName: string; - filePath: string; - importRoot: string; - packageDirectory?: string; -}; - -async function findPythonEntryTarget(projectDir: string): Promise { - const root = existsSync(path.join(projectDir, "src")) ? path.join(projectDir, "src") : projectDir; - const entries = (await readdir(root, { withFileTypes: true })).sort((a, b) => - a.name.localeCompare(b.name), - ); - for (const entry of entries) { - if (!entry.isDirectory() || !/^[A-Za-z_]\w*$/.test(entry.name)) continue; - const init = path.join(root, entry.name, "__init__.py"); - if (existsSync(init)) { - const entryModule = ["main.py", "app.py", "__main__.py"].find((name) => - existsSync(path.join(root, entry.name, name)), - ); - if (entryModule) { - return { - moduleName: `${entry.name}.${entryModule.slice(0, -3)}`, - filePath: path.join(root, entry.name, entryModule), - importRoot: root, - packageDirectory: path.dirname(init), - }; - } - return { - moduleName: entry.name, - filePath: init, - importRoot: root, - packageDirectory: path.dirname(init), - }; - } - } - for (const entry of entries) { - if (!/^[A-Za-z_]\w*$/.test(entry.name.replace(/\.py$/, ""))) continue; - if ( - entry.isFile() && - entry.name.endsWith(".py") && - !/^(__init__|setup|conftest)\.py$/.test(entry.name) - ) { - return { - moduleName: entry.name.slice(0, -3), - filePath: path.join(root, entry.name), - importRoot: root, - }; - } - if (entry.isDirectory()) { - const module = (await readdir(path.join(root, entry.name))) - .filter((name) => /^[A-Za-z_]\w*\.py$/.test(name) && name !== "__init__.py") - .sort()[0]; - if (module) { - return { - moduleName: `${entry.name}_${module.slice(0, -3)}`, - filePath: path.join(root, entry.name, module), - importRoot: path.join(root, entry.name), - }; - } - } - } - return null; -} - -function pythonFileImportCommand(target: PythonEntryTarget) { - const moduleName = JSON.stringify(target.moduleName); - const filePath = JSON.stringify(target.filePath); - const importRoot = JSON.stringify(target.importRoot); - if (target.moduleName.includes(".")) { - return [ - "import importlib, sys", - `sys.path.insert(0, ${importRoot})`, - `importlib.import_module(${moduleName})`, - ].join("; "); - } - const packageLocations = target.packageDirectory - ? `, submodule_search_locations=[${JSON.stringify(target.packageDirectory)}]` - : ""; - return [ - "import importlib.util, pathlib, sys", - `p = pathlib.Path(${filePath})`, - `sys.path.insert(0, ${importRoot})`, - `s = importlib.util.spec_from_file_location(${moduleName}, p${packageLocations})`, - "assert s is not None and s.loader is not None", - "m = importlib.util.module_from_spec(s)", - `sys.modules[${moduleName}] = m`, - "s.loader.exec_module(m)", - ].join("; "); -} - -function runGoTidyAdvisory(projectDir: string) { - return Effect.gen(function* () { - const tidy = yield* commandStep("go", ["mod", "tidy", "-diff"], projectDir); - return { ...tidy, command: "go mod tidy (advisory diff)" }; - }); -} - -async function readOptional(filePath: string) { - try { - return await readFile(filePath, "utf8"); - } catch { - return null; - } -} diff --git a/scripts/scaffbench/validation/java.ts b/scripts/scaffbench/validation/java.ts deleted file mode 100644 index b7e3ad8be..000000000 --- a/scripts/scaffbench/validation/java.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateJavaProject } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/planner.ts b/scripts/scaffbench/validation/planner.ts deleted file mode 100644 index 50e24c141..000000000 --- a/scripts/scaffbench/validation/planner.ts +++ /dev/null @@ -1 +0,0 @@ -export { findProjectDir, archiveProjectSource } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/python.ts b/scripts/scaffbench/validation/python.ts deleted file mode 100644 index 81a110398..000000000 --- a/scripts/scaffbench/validation/python.ts +++ /dev/null @@ -1 +0,0 @@ -export { validatePythonProject } from "@scaffbench/validation/index"; diff --git a/scripts/scaffbench/validation/shared.ts b/scripts/scaffbench/validation/shared.ts deleted file mode 100644 index e87beff80..000000000 --- a/scripts/scaffbench/validation/shared.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { readdir } from "node:fs/promises"; -import path from "node:path"; - -/** Dependency/build trees shared by validation discovery and source metrics. */ -export const PROJECT_WALK_SKIP_DIRECTORIES = new Set([ - "node_modules", - "target", - ".git", - "deps", - "_build", - "vendor", - "Pods", - ".venv", - ".dart_tool", - ".gradle", - "obj", - "bin", - "dist", - "build", - ".next", - ".expo", - ".svelte-kit", - ".output", - ".nuxt", - ".vercel", - ".turbo", - ".wrangler", - "coverage", -]); - -export function parseJsonc(raw: string) { - const withoutLineComments = raw - .split("\n") - .filter((line) => !line.trim().startsWith("//")) - .join("\n"); - const withoutTrailingCommas = withoutLineComments.replace(/,\s*([}\]])/g, "$1"); - try { - return JSON.parse(withoutTrailingCommas); - } catch { - return null; - } -} - -export async function walk(dir: string, visit: (filePath: string) => Promise) { - // Validation also ignores Turbo's cache. Source metrics intentionally do not: - // their exclusion contract is the exact shared set above. - const skip = new Set([...PROJECT_WALK_SKIP_DIRECTORIES, ".turbo"]); - let entries; - try { - entries = await readdir(dir, { withFileTypes: true }); - } catch { - return; - } - for (const entry of entries) { - if (skip.has(entry.name)) continue; - const next = path.join(dir, entry.name); - if (entry.isDirectory()) { - await walk(next, visit); - } else if (entry.isFile()) { - await visit(next); - } - } -} diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index 8e9e4a4bc..91963df8c 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -5,7 +5,6 @@ "paths": { "@actions/*": ["../.github/actions/*"], "@root/*": ["../*"], - "@scaffbench/*": ["./scaffbench/*"], "@scripts/*": ["./*"], "@testing/*": ["../testing/*"], "@web/*": ["../apps/web/src/*"] diff --git a/scripts/validation/build-verified-combinations.ts b/scripts/validation/build-verified-combinations.ts index 66c637872..04c6e5193 100644 --- a/scripts/validation/build-verified-combinations.ts +++ b/scripts/validation/build-verified-combinations.ts @@ -5,7 +5,6 @@ import { getPresetCombos } from "@testing/lib/presets"; import { evaluatePublishedPackageEvidence, evaluateReleaseGuardEvidence, - evaluateScaffbenchEvidence, evaluateSmokeEvidence, REQUIRED_MANAGERS, SOURCE_EVIDENCE_MAX_AGE_MS, @@ -28,69 +27,6 @@ type SmokeResult = { totalDurationMs?: number; }; -type ScaffbenchSpec = { - id: string; - title: string; - family?: string; - canonicalFlags?: string[]; -}; - -type ScaffbenchStep = { - command: string; - exitCode: number | null; - status?: "ran" | "skip" | "na"; - timedOut?: boolean; -}; - -type ScaffbenchResult = { - specId: string; - specTitle?: string; - path?: string; - trial?: number; - validation?: { - projectExists?: boolean; - deferred?: boolean; - install?: ScaffbenchStep; - build?: ScaffbenchStep; - checkTypes?: ScaffbenchStep; - lint?: ScaffbenchStep; - format?: ScaffbenchStep; - test?: ScaffbenchStep; - doctor?: ScaffbenchStep; - route?: ScaffbenchStep; - steps?: Record; - }; - stackScore?: { - matched: number; - total: number; - percent: number; - }; - failureTags?: string[]; -}; - -type ScaffbenchSummary = { - generatedAt?: string; - options?: { - specs?: string[]; - paths?: string[]; - qualityGate?: boolean; - doctorCheck?: boolean; - routeCheck?: boolean; - }; - metadata?: { - evidenceSchemaVersion?: number; - gitHead?: string; - gitBranch?: string; - workspaceClean?: boolean; - bfGeneratorVersion?: string; - environmentQualified?: boolean; - generatorSource?: string; - generatorGitHead?: string; - }; - specs?: ScaffbenchSpec[]; - results?: ScaffbenchResult[]; -}; - type ReleaseGuardStep = { command: string; durationMs?: number; @@ -147,14 +83,6 @@ const SMOKE_INPUTS: SmokeInput[] = [ preset: "pr-core", }, ]; -const SCAFFBENCH_INPUTS = [ - { - label: "ScaffBench 2", - path: "testing/.tmp-scaffbench-2/summary.json", - expectedSpecIds: ["ai-search-workbench"], - }, -]; - async function readJson(filePath: string): Promise { try { return JSON.parse(await readFile(filePath, "utf8")) as T; @@ -221,19 +149,6 @@ type VerifiedClaimSummary = { rerunCommand: string; failureHint: string; }>; - scaffbench: Array<{ - label: string; - source: string; - pass: number; - total: number; - current?: boolean; - reasons?: string[]; - environmentQualified?: boolean; - ownerArea: string; - actionLinks: ActionLink[]; - rerunCommand: string; - failureHint: string; - }>; releaseGuard: { source: string; pass: number; @@ -373,73 +288,6 @@ function smokeStatus(result: SmokeResult): EvidenceStatus { return result.steps.some((step) => !step.success && !step.skipped) ? "partial-pass" : "pass"; } -function formatScaffbenchSteps(result: ScaffbenchResult): string { - const validation = result.validation; - if (!validation) { - return "no validation payload"; - } - - const orderedSteps: Array = validation.steps - ? Object.entries(validation.steps) - : [ - ["install", validation.install], - ["build", validation.build], - ["typecheck", validation.checkTypes], - ["lint", validation.lint], - ["format", validation.format], - ["test", validation.test], - ["doctor", validation.doctor], - ["route", validation.route], - ]; - - return orderedSteps - .filter(([, step]) => Boolean(step)) - .map(([name, step]) => { - if (step?.status === "na") { - return `${name}: n/a`; - } - - if (step?.status === "skip") { - return `${name}: skipped`; - } - - if (step?.timedOut) { - return `${name}: timed out`; - } - - return `${name}: ${step?.exitCode === 0 ? "pass" : "fail"}`; - }) - .join("
    "); -} - -function scaffbenchResultStatus(result: ScaffbenchResult): "pass" | "fail" { - if (result.failureTags && result.failureTags.length > 0) { - return "fail"; - } - - if (result.validation?.deferred || result.validation?.projectExists === false) { - return "fail"; - } - - const steps = result.validation?.steps - ? Object.values(result.validation.steps) - : [ - result.validation?.install, - result.validation?.build, - result.validation?.checkTypes, - result.validation?.lint, - result.validation?.format, - result.validation?.test, - result.validation?.doctor, - result.validation?.route, - ].filter(Boolean); - - return steps.length > 0 && - steps.every((step) => step?.status === "ran" && step.exitCode === 0 && step.timedOut !== true) - ? "pass" - : "fail"; -} - function smokeCommandMap(): Map { const commands = new Map(); @@ -494,11 +342,6 @@ export async function readSmokeResults(input: SmokeInput): Promise<{ return { results: [...orderedResults, ...extraResults], sources }; } -function commandFromScaffbenchSpec(spec: ScaffbenchSpec): string { - const flags = spec.canonicalFlags?.join(" ") ?? ""; - return `bun create better-fullstack@latest ${spec.id} ${flags}`.trim(); -} - function releaseGuardOwner(command: string): string { if (command.includes("validate-plugin-bundle")) { return "Plugin bundle packaging"; @@ -710,112 +553,6 @@ async function renderSmokeSection(claimSummary: VerifiedClaimSummary): Promise { - const sections: string[] = ["## ScaffBench Evidence"]; - - for (const [inputIndex, input] of SCAFFBENCH_INPUTS.entries()) { - const claim = claimSummary.scaffbench[inputIndex]; - const summary = await readJson(input.path); - if (!summary) { - sections.push(`No ScaffBench evidence found at ${code(input.path)}.`); - continue; - } - - const runCount = summary.results?.length ?? 0; - const specCount = summary.specs?.length ?? 0; - const metadata = summary.metadata; - const sourceSummary = [ - `Source: ${code(input.path)}.`, - `Specs: ${specCount}.`, - `Runs: ${runCount}.`, - metadata?.gitHead ? `Git head: ${code(metadata.gitHead.slice(0, 12))}.` : "", - metadata?.bfGeneratorVersion ? `Generator: ${code(metadata.bfGeneratorVersion)}.` : "", - typeof metadata?.environmentQualified === "boolean" - ? `Environment qualified: ${metadata.environmentQualified ? "yes" : "no"}.` - : "", - ] - .filter(Boolean) - .join(" "); - - sections.push(`### ${input.label}`); - sections.push(sourceSummary); - - if (runCount > 0) { - sections.push( - [ - "| Status | Spec | Path | Trial | Owner area | Stack score | Validation steps | Failure tags | Action links |", - "| --- | --- | --- | --- | --- | --- | --- | --- | --- |", - ...(summary.results ?? []) - .map((result) => { - const spec = summary.specs?.find((candidate) => candidate.id === result.specId); - const family = spec?.family ?? "multi-ecosystem"; - const ownerArea = ownerAreaForEcosystem(family); - const status = claim?.current === true ? scaffbenchResultStatus(result) : "fail"; - const stackScore = result.stackScore - ? `${result.stackScore.matched}/${result.stackScore.total} (${result.stackScore.percent}%)` - : "n/a"; - - return [ - statusLabel(status), - code(result.specId), - result.path ?? "n/a", - result.trial ?? "n/a", - code(ownerArea), - stackScore, - formatScaffbenchSteps(result), - result.failureTags?.join(", ") || "none", - actionLinksCell( - compactActionLinks([ - repoActionLink("source", input.path), - { label: "owner", href: repoUrl(ownerArea) }, - { label: "runner", href: repoUrl("scripts/benchmarks/scaffbench-v2.ts") }, - ]), - "bun run scaffbench:2:canonical", - ), - ] - .map(escapeTableCell) - .join(" | "); - }) - .map((row) => `| ${row} |`), - ].join("\n"), - ); - } - - const matrixSpecs = summary.specs ?? []; - if (matrixSpecs.length > 0) { - sections.push("### ScaffBench Spec Matrix"); - sections.push( - runCount === 0 - ? "These rows are matrix-only evidence: they prove the benchmark has a spec and canonical command, not that validation passed." - : "These rows list the canonical commands backing the benchmark specs.", - ); - sections.push( - [ - "| Status | Spec | Family | Owner area | Command |", - "| --- | --- | --- | --- | --- |", - ...matrixSpecs - .map((spec) => { - const status = runCount === 0 ? "matrix-only" : "configured"; - const family = spec.family ?? "multi-ecosystem"; - return [ - statusLabel(status), - code(spec.id), - family, - code(ownerAreaForEcosystem(family)), - code(commandFromScaffbenchSpec(spec)), - ] - .map(escapeTableCell) - .join(" | "); - }) - .map((row) => `| ${row} |`), - ].join("\n"), - ); - } - } - - return sections.join("\n\n"); -} - async function renderReleaseGuardSection( claimSummary: VerifiedClaimSummary, currentGitHead?: string, @@ -1016,7 +753,6 @@ async function buildVerifiedClaimSummary( currentGitHead?: string, ): Promise { const smoke: VerifiedClaimSummary["smoke"] = []; - const scaffbench: VerifiedClaimSummary["scaffbench"] = []; const evidenceTimestamps: number[] = []; const recordEvidenceTimestamp = (value: unknown) => { const parsed = typeof value === "string" ? Date.parse(value) : Number.NaN; @@ -1069,29 +805,6 @@ async function buildVerifiedClaimSummary( }); } - for (const input of SCAFFBENCH_INPUTS) { - const summary = await readJson(input.path); - if (summary) recordEvidenceTimestamp(summary.generatedAt); - const results = summary?.results ?? []; - const ownerArea = "packages/template-generator/templates"; - scaffbench.push({ - label: input.label, - source: input.path, - pass: results.filter((result) => scaffbenchResultStatus(result) === "pass").length, - total: input.expectedSpecIds.length, - environmentQualified: summary?.metadata?.environmentQualified, - ownerArea, - actionLinks: compactActionLinks([ - repoActionLink("source", input.path), - { label: "runner", href: repoUrl("scripts/benchmarks/scaffbench-v2.ts") }, - { label: "owner", href: repoUrl(ownerArea) }, - ]), - rerunCommand: "bun run scaffbench:2:canonical", - failureHint: - "Inspect failureTags and validation steps in the ScaffBench summary, then follow the owner area for the stack family.", - }); - } - const releaseSummary = await readJson(RELEASE_GUARD_INPUT); const publishedPackageSummary = await readJson(PUBLISHED_PACKAGE_INPUT); @@ -1107,7 +820,6 @@ async function buildVerifiedClaimSummary( publishedPackage: REQUIRED_MANAGERS.length, }, smoke, - scaffbench, releaseGuard: releaseSummary ? { source: RELEASE_GUARD_INPUT, @@ -1214,19 +926,6 @@ export type VerifiedCombinationSummary = { rerunCommand: string; failureHint: string; }>; - scaffbench: Array<{ - label: string; - source: string; - pass: number; - total: number; - current?: boolean; - reasons?: string[]; - environmentQualified?: boolean; - ownerArea: string; - actionLinks: VerifiedCombinationActionLink[]; - rerunCommand: string; - failureHint: string; - }>; releaseGuard: { source: string; pass: number; @@ -1281,12 +980,6 @@ async function enforceCurrentEvidence( const verdict = evaluateSmokeEvidence(await readJson(input.path), expected, context); Object.assign(summary.smoke[index]!, verdict); } - for (const [index, input] of SCAFFBENCH_INPUTS.entries()) { - const raw = await readJson(input.path); - const expected = input.expectedSpecIds; - const verdict = evaluateScaffbenchEvidence(raw, expected, context); - Object.assign(summary.scaffbench[index]!, verdict); - } if (summary.releaseGuard) { Object.assign( summary.releaseGuard, @@ -1339,10 +1032,9 @@ async function main(): Promise { `Generated by ${code("bun run build:verified-combinations")}.`, `Last generated: ${generatedAt}.`, ].join(" "), - "A row marked Pass has green evidence in the generated input file. Partial pass means the smoke harness reported an overall pass while one or more non-gating steps failed. Failed rows are intentionally visible. Matrix-only and Configured rows are coverage commitments, not green merge claims. ScaffBench is a separate model benchmark and never contributes to product release verification.", + "A row marked Pass has green evidence in the generated input file. Partial pass means the smoke harness reported an overall pass while one or more non-gating steps failed. Failed rows are intentionally visible. Matrix-only and Configured rows are coverage commitments, not green merge claims. Fixproof is a separate coding-agent benchmark and never contributes to product release verification.", renderCurrentClaim(claimSummary), await renderSmokeSection(claimSummary), - await renderScaffbenchSection(claimSummary), await renderReleaseGuardSection(claimSummary, currentGitHead), await renderPublishedPackageSection(claimSummary, expectedPackageVersion), ]; diff --git a/scripts/validation/verified-combinations-evidence.test.ts b/scripts/validation/verified-combinations-evidence.test.ts index 0580ef044..3cc9ae6b0 100644 --- a/scripts/validation/verified-combinations-evidence.test.ts +++ b/scripts/validation/verified-combinations-evidence.test.ts @@ -4,7 +4,6 @@ import { EVIDENCE_SCHEMA_VERSION, evaluatePublishedPackageEvidence, evaluateReleaseGuardEvidence, - evaluateScaffbenchEvidence, evaluateSmokeEvidence, type EvidenceReason, } from "@scripts/verified-combinations/evidence"; @@ -206,115 +205,6 @@ describe("source-bound evidence", () => { }); }); -describe("ScaffBench evidence", () => { - const validResult = { - specId: "spec", - failureTags: [], - validation: { - projectExists: true, - deferred: false, - steps: { build: { status: "ran", exitCode: 0, timedOut: false } }, - }, - }; - const valid = { - generatedAt: source.generatedAt, - metadata: { - evidenceSchemaVersion: EVIDENCE_SCHEMA_VERSION, - gitHead: HEAD, - workspaceClean: true, - environmentQualified: true, - generatorSource: "workspace-local", - generatorGitHead: HEAD, - bfGeneratorVersion: "2.5.0", - }, - results: [validResult], - }; - - it("accepts qualified execution with real passing steps", () => { - expect(evaluateScaffbenchEvidence(valid, ["spec"], context)).toEqual({ - current: true, - pass: 1, - total: 1, - reasons: [], - }); - }); - - it.each([ - ["unqualified", { metadata: { environmentQualified: false } }, "environment-unqualified"], - ["an unbound generator", { metadata: { generatorSource: "registry" } }, "generator-unbound"], - [ - "the wrong generator version", - { metadata: { bfGeneratorVersion: "2.4.0" } }, - "wrong-package-version", - ], - [ - "empty", - { results: [{ ...validResult, validation: { projectExists: true, steps: {} } }] }, - "no-executed-steps", - ], - [ - "deferred", - { - results: [ - { - ...validResult, - validation: { deferred: true, steps: validResult.validation.steps }, - }, - ], - }, - "deferred-validation", - ], - [ - "skipped", - { - results: [ - { - ...validResult, - validation: { steps: { build: { status: "skip", exitCode: null } } }, - }, - ], - }, - "skipped-validation", - ], - [ - "failed", - { - results: [ - { ...validResult, validation: { steps: { build: { status: "ran", exitCode: 1 } } } }, - ], - }, - "failed-validation", - ], - [ - "unknown step status", - { - results: [{ ...validResult, validation: { steps: { build: { exitCode: 0 } } } }], - }, - "failed-validation", - ], - [ - "timed out", - { - results: [ - { - ...validResult, - validation: { steps: { build: { status: "ran", exitCode: 0, timedOut: true } } }, - }, - ], - }, - "failed-validation", - ], - ])("rejects %s validation", (_, override, reason) => { - const result = evaluateScaffbenchEvidence( - { ...valid, ...override, metadata: { ...valid.metadata, ...(override as any).metadata } }, - ["spec"], - context, - ); - expect(result.pass).toBe(0); - expect(result.reasons).toContain(reason as EvidenceReason); - }); -}); - describe("published-package evidence", () => { const results = ["bun", "npm", "pnpm"].map((manager) => ({ manager, status: "pass" })); const publishedContext = { now: NOW, expectedVersion: "2.5.0" }; diff --git a/scripts/verified-combinations/evidence.ts b/scripts/verified-combinations/evidence.ts index 8ee9d3a99..e906a2bf0 100644 --- a/scripts/verified-combinations/evidence.ts +++ b/scripts/verified-combinations/evidence.ts @@ -13,12 +13,8 @@ export type EvidenceReason = | "stale-timestamp" | "unsuccessful" | "incomplete-rows" - | "environment-unqualified" | "no-executed-steps" - | "deferred-validation" - | "skipped-validation" | "failed-validation" - | "generator-unbound" | "wrong-package-version" | "wrong-package-identity" | "wrong-registry" @@ -36,8 +32,6 @@ const OUTCOME_REASONS: ReadonlySet = new Set([ "unsuccessful", "failed-validation", "no-executed-steps", - "deferred-validation", - "skipped-validation", ]); export type SourceEvidenceContext = { @@ -208,90 +202,6 @@ export function evaluateReleaseGuardEvidence( return verdict(expectedCommands.length, pass, [...new Set(reasons)]); } -export function evaluateScaffbenchEvidence( - input: unknown, - expectedSpecIds: readonly string[], - context: SourceEvidenceContext, -): EvidenceVerdict { - if (!input || typeof input !== "object" || Array.isArray(input)) { - return verdict( - expectedSpecIds.length, - 0, - input === null || input === undefined ? ["missing"] : ["unrecognized-version"], - ); - } - const summary = input as Record; - const metadata = summary.metadata ?? {}; - const results = Array.isArray(summary.results) ? summary.results : []; - const reasons = sourceReasons( - { - schemaVersion: metadata.evidenceSchemaVersion, - generatedAt: summary.generatedAt, - gitHead: metadata.gitHead, - workspaceClean: metadata.workspaceClean, - overallSuccess: results.length > 0, - }, - context, - ); - if (metadata.environmentQualified !== true) reasons.push("environment-unqualified"); - if ( - metadata.generatorSource !== "workspace-local" || - metadata.generatorGitHead !== metadata.gitHead - ) { - reasons.push("generator-unbound"); - } - if ( - typeof context.currentPackageVersion !== "string" || - metadata.bfGeneratorVersion !== context.currentPackageVersion - ) { - reasons.push("wrong-package-version"); - } - const resultSpecIds = new Set(results.map((result: any) => result.specId)); - if ( - expectedSpecIds.some((id) => !resultSpecIds.has(id)) || - results.length !== expectedSpecIds.length - ) { - reasons.push("incomplete-rows"); - } - let passed = 0; - for (const result of results) { - const validation = result?.validation; - const steps = - validation && typeof validation.steps === "object" - ? (Object.values(validation.steps).filter(Boolean) as Array>) - : [ - validation?.install, - validation?.build, - validation?.checkTypes, - validation?.lint, - validation?.format, - validation?.test, - validation?.doctor, - validation?.route, - ].filter(Boolean); - const executed = steps.filter((step) => step.status === "ran"); - if (validation?.deferred) reasons.push("deferred-validation"); - if (executed.length === 0) reasons.push("no-executed-steps"); - if (steps.some((step) => step.status === "skip" || step.status === "na")) - reasons.push("skipped-validation"); - if ( - result?.failureTags?.length > 0 || - validation?.projectExists === false || - steps.some((step) => step.status !== "ran" || step.exitCode !== 0 || step.timedOut === true) - ) - reasons.push("failed-validation"); - if ( - !validation?.deferred && - validation?.projectExists !== false && - executed.length > 0 && - steps.every((step) => step.status === "ran" && step.exitCode === 0 && !step.timedOut) && - !(result?.failureTags?.length > 0) - ) - passed += 1; - } - return verdict(expectedSpecIds.length, passed, [...new Set(reasons)]); -} - export function evaluatePublishedPackageEvidence( input: unknown, context: { now: Date; expectedVersion: string; maxAgeMs?: number }, diff --git a/testing/scaffbench-3-glm-5-3-flash-2026-08-22.md b/testing/scaffbench-3-glm-5-3-flash-2026-08-22.md deleted file mode 100644 index ad3278f24..000000000 --- a/testing/scaffbench-3-glm-5-3-flash-2026-08-22.md +++ /dev/null @@ -1,166 +0,0 @@ -# ScaffBench 3: GLM 5.3 Flash at high effort - -First run of the reset suite. The model ran under opencode's stealth alias "Ox -Alpha Free (Unlimited)" and has since been identified as Z.ai's GLM 5.3 Flash. It -is listed in the CLI as `opencode/x-preview-f-free`. The alias and the id are only -linked through the `name` field in the models.dev catalog, which is worth knowing -before anyone goes looking for an `ox-alpha` or `glm-5.3-flash` id on that -endpoint. The `opencode-go/ox-alpha-free` id is a different door onto the -subscription endpoint and needs a funded workspace. - -Run on a dedicated Linux box (6 cores, 15 GiB), not a laptop. Protocol: harness -3.1.0, suite 3.0, prompt 2026-08-21-scaffbench-3.1, validation cache v8, resource -profile low-2w-v1, repeats 1, prompt path only. - -## Headline - -Pass@1 is 8 of 13, or 62%, with a 95% Wilson interval of 36 to 82. Index 69 under -the 75/25 formula the suite launched with; 58 under the graded, difficulty-weighted -index adopted on 2026-08-27 (see the Scoring section of -`docs/guidelines/scaffbench-benchmark.md`). -Cost was zero across every spec. - -The gap between wiring and compiling is the story. Stack score averages 90%, so -the model picks and imports the right libraries almost every time. Then a third of -the projects fail to typecheck or build. Quality, the stricter tier that also wants -lint, format and test green, lands at 15%: two projects out of thirteen. - -Generation averaged 27 minutes per spec, median 28, p95 49, at 30,300 output -tokens per spec. - -## Stack score against Pass@1 - -``` -spec stack wired pass@1 -dotnet-blazor-cqrs 24/24 ████████████████████ 100% pass -go-realtime-api 16/16 ████████████████████ 100% pass -ts-svelte-edge-orpc 13/13 ████████████████████ 100% pass -elixir-broadway-absinthe 20/20 ████████████████████ 100% pass -rust-leptos-axum 23/24 ███████████████████░ 96% pass -multi-ts-go-grpc 21/22 ███████████████████░ 95% pass -multi-dotnet-ops 17/18 ███████████████████░ 94% pass -java-spring-jooq-keycloak 26/28 ███████████████████░ 93% pass -react-native-expo 13/13 ████████████████████ 100% FAIL typecheck -frontier-effect-eventsourcing 14/14 ████████████████████ 100% FAIL build -frontier-polyglot-proto 10/10 ████████████████████ 100% FAIL buf prerequisite -python-ingestion-api 15/16 ███████████████████░ 94% FAIL typecheck -ai-search-workbench 26/27 ███████████████████░ 96% FAIL build (vite+ clash) -``` - -Three of the five failures scored a perfect stack. The model assembled everything -the spec asked for and still could not make it compile. - -## Full results - -| spec | pass@1 | stack | failed steps | -| --- | --- | --- | --- | -| java-spring-jooq-keycloak | pass | 26/28 | none | -| dotnet-blazor-cqrs | pass | 24/24 | none | -| go-realtime-api | pass | 16/16 | format | -| ts-svelte-edge-orpc | pass | 13/13 | lint, format, test | -| elixir-broadway-absinthe | pass | 20/20 | test | -| multi-ts-go-grpc | pass | 21/22 | lint, format, test | -| multi-dotnet-ops | pass | 17/18 | lint | -| rust-leptos-axum | pass | 23/24 | format | -| react-native-expo | fail | 13/13 | typecheck | -| python-ingestion-api | fail | 15/16 | typecheck | -| frontier-effect-eventsourcing | fail | 14/14 | build | -| ai-search-workbench | fail | 26/27 | build | -| frontier-polyglot-proto | fail | 10/10 | prerequisite: buf generate | - -Failure tags across the run: test-failed 9, format-failed 8, lint-failed 7, -stack-mismatch 6, validation-failed 5, typecheck-failed 3, build-failed 1, -project-not-found 1. - -## Both disputed failures are settled, and neither changes the score - -Both needed settling before this `ranked` row went near the board. Neither -survives as an exclusion, so Pass@1 stands at 8 of 13, or 62%, with no asterisk. - -### ai-search-workbench: a dropped request, then a real failure - -The first attempt ran two steps that made four tool calls (bash, todowrite, bash, -websearch), emitted 588 output tokens, then returned a third step with -`reason: "unknown"` and zero tokens in and out. The process exited 0 after two -minutes with no project on disk. That is the opencode dead-request signature -already seen with Kimi K3, where a dropped request arrives as a zero-usage -unknown rather than an error. - -The re-run settled it. This time the model worked for 31 minutes, exited on -`stop`, emitted 46,337 output tokens and wired 26 of the 27 requested libraries. -Install passed and `build` failed: - -``` -$ vp run build -error: Failed to load task graph -* Task sb21-ai-search-workbench#dev conflicts with a package.json script - of the same name. Remove the script from package.json or rename the task -``` - -The model put the project on Vite+ as the spec demands, wrote `"dev": "vp run dev"` -into the root package.json, and also defined a `dev` task in the Vite+ config. -Vite+ refuses to load a task graph when a task name collides with a script name, -so every `vp` command dies, which is why typecheck, lint, format and test never -ran. One naming collision took down the whole toolchain. That is a fair failure -and a good trap: the model knows the tool's surface and not its constraints. - -The harness bug behind the first attempt is still real and still unfixed. -`agents/opencode.ts:109` guards for the dead-request signature, but the condition -is `stepReason === "unknown" && outputTokens === 0 && !sawTool && !sawAssistantText`. -It only catches sessions that die before doing anything. That one had already -worked, so it fell through and scored as a model failure. Any opencode run that -dies mid-flight is mis-scored the same way, which is the expensive case: it looks -exactly like a real failure in published data. - -### frontier-polyglot-proto: a real failure, reached the long way - -The spec runs `buf generate` as a prerequisite. It failed with -`plugin protoc-gen-ts_proto: executable file not found in $PATH`, fail-fast -stopped the run, and install never executed. That prerequisite ordering is a -harness bug in its own right: - -- The model declared `ts-proto` as a devDependency, so the plugin binary lands in - `node_modules/.bin`. -- Bare `buf generate` fails even after `bun install`, because buf resolves - `local:` plugins through `$PATH` only. -- With `node_modules/.bin` on `$PATH` it succeeds, exit 0, and writes `gen/ts`. - -So `prerequisiteCommands` running before install makes any npm-based local codegen -plugin impossible to pass, whatever the model writes. Only exclusively-remote -plugins survive. That will cost a better model a spec later, so it is worth fixing. - -It did not cost this one a spec. Building each half by hand, after install and a -working `buf generate`, shows the project does not hold up: - -| module | result | -| --- | --- | -| `gateway/go-gateway` | builds, exit 0 | -| `services/rust-core` (cargo) | builds, exit 0 | -| `clients/ts-client` (tsc) | typechecks, exit 0 | -| `gen/go` | fails | - -`gen/go` is its own module. Its `go.mod` requires grpc and protobuf, but the model -never wrote a `go.sum`, so building it standalone fails with `missing go.sum entry` -for every generated import. The gateway gets away with it by pulling `gen/go` -through a `replace` directive and covering those dependencies in its own `go.sum`. -`validation/index.ts:501` calls `findManifestRoots(projectDir, ["go.mod"])` and -validates every Go root it finds, so the validator builds `gen/go` directly and it -fails. Fixing the prerequisite ordering would only move the failure one step later. - -## Toolchains - -bun 1.4.0, node 24.19.0, rustc 1.98.0, cargo 1.98.0, go 1.27.0, dotnet 10.0.400, -python 3.12.3, uv 0.12.5, java 21.0.12 (Temurin), maven 3.9.16, gradle 9.7.1, -elixir 1.20.3 on OTP 29, buf 1.72.0, protoc 35.1, psql 16.15. - -## Not done yet - -- gpt-5.6-luna at high effort is running now for comparison. -- The publication gates from the pre-run audit are untouched: the three canonical - runs are not re-recorded under cache v8, dotnet-blazor-cqrs has no canonical, and - the weak-versus-strong calibration pass has not run. Nothing here belongs on the - public board until those close. -- Only `low`, `high` and `max` are real efforts for this model. The provider - rejects `medium` and `xhigh`, and opencode drops an unknown `--variant` silently - instead of erroring, so a run labelled `medium` would quietly record the - provider default. diff --git a/testing/scaffbench-hardening-fixes-spec-2026-07-17.md b/testing/scaffbench-hardening-fixes-spec-2026-07-17.md deleted file mode 100644 index 57d67f52d..000000000 --- a/testing/scaffbench-hardening-fixes-spec-2026-07-17.md +++ /dev/null @@ -1,141 +0,0 @@ -# ScaffBench hardening - review-fix spec (round 2, 2026-07-17) - -Union of two independent adversarial reviews (Fable 5 + GPT-5.6 Sol) of commit -a72440690. Fix ALL items. Same scope rules as round 1: only `scripts/**` and -script tests; do NOT touch `apps/web/**` (parallel stream owns it); no sweeps; -don't modify `testing/llm-benchmarks/**`. Acceptance: tsc scripts scope clean, -`bun test scripts/` green, a NEW regression test per item that asserts the -FAILURE scenario described (not just the helper's happy path). Append a -"Round 2" section to `testing/scaffbench-hardening-report-2026-07-17.md`. -End your final message with the exact line: FIXES COMPLETE - -## A. Multi-trial aggregation drops no-project failures (both reviews, High) -`build-scaffbench-2-1-data.ts:112` - `scoredTrials = min(aggregate.scoredRuns, -measurable.length)` erases scored trials that produced no project/steps. -{no-project fail, pass, pass} → passRate 100. Fix: derive scored/pass counts -from EVERY raw trial's persisted outcome + corePass, counting scored -non-measurable trials as failures; assert exact equality with summary -aggregates and THROW on mismatch (never repair with min/max). Test must use a -failed trial with NO steps. - -## B. Splice quality normalization (Fable #2) -`splice-scaffbench-2-1.ts:113-114` - `qualityPassCount: existing.qualityPassCount -?? (cell.fullPass ? 1 : 0)` turns fullPass:null into a measured 0. Fix: -null-propagate (`cell.fullPass === null ? null : ...`) for both count and rate. - -## C. Prompt/spec install-policy contradiction (Sol #3, High) -`prompts.ts:41` allows installs; spec BODIES still forbid them (e.g. -`specs/frontier-polyglot-proto.ts:19`; grep all specs for install prohibitions). -Policy decision (final): self-verification is ALLOWED. Remove/reword every -"do not install" style line in spec bodies so they no longer contradict the -base prompt; where a spec means "the CLI scaffold step must use --no-install", -say exactly that. Add a prompt snapshot test asserting the generated prompt -for each pathMode contains no contradictory install instructions. - -## D. Idle-kill is adapter-unsafe (both, High) -`agents/command.ts:43` counts only stdout bytes; all adapters enable it; agy -buffers everything. Fix: make idle timeout an adapter capability - enable ONLY -for verified streaming JSONL adapters (claude, codex, opencode, kilo), DISABLE -for agy. Count BOTH stdout and stderr bytes as activity. Suspend idle -enforcement while a tool call is in flight (tool-start event seen without its -completion) - a silent 30-min build inside a tool call must not be killed -(the new prompt explicitly invites installs/builds). Test: buffered-adapter -simulation (no stdout for > idle window, then output) survives when idle -disabled; in-flight tool-call suspension covered. - -## E. Transient-network classification too loose (both, High) -`validation/classification.ts`: -- `\b5\d\d\b` with optional HTTP prefix matches "531 packages installed" when - any registry URL appears in the tail. Require adjacency to HTTP/status/error - context (e.g. /(?:HTTP|status(?: code)?|error)\D{0,10}5\d\d/i) AND a - registry/package-manager fetch context. -- `\b429\b` matches version substrings like "0.429.1" (dots are word - boundaries). Require the same HTTP/status/too-many-requests context. -- The 404 short-circuit at :23 returns false even when a genuine 503 is also - present - only treat 404/version-not-found as model-owned when NO transient - signature matches a DIFFERENT line. -- `isRecurringTransientFailure` (:37-43) must require actual retry evidence - (the step was retried and failed transiently again), not the raw signature - alone - otherwise pre-2.2 results re-summarized get silently reclassified. - Keep raw signature only for cache exclusion. -- `scoring.ts:556` provider-infra regex: add \b anchors and require error-ish - context; "capacity" inside prose or "14293" must not match. Tests: lifecycle - script printing 429/DNS errors twice → model failure; "531 packages - installed" → not transient; mixed 404+503 → transient; "0.429.1" → not. - -## F. opencode zero-usage vs refusal (Sol #8) -`agents/opencode.ts` - track assistant text/refusal events; the provider-infra -signature must require reason:"unknown" + zero usage + no tools + NO assistant -text. A refusal/moderation with text is a model failure. Negative test. - -## G. Workspace membership + solution coverage (both, High) -`validation/index.ts` - `dropNestedRoots` never parses membership; .NET builds -only `solutions[0]`. Fix: parse actual membership (bun/npm `workspaces` globs, -`[workspace] members` in Cargo.toml, sln project references; python: keep -current heuristic but emit steps). A discovered manifest not covered by a -parent workspace/solution gets validated independently; when it cannot be -(overflow, unsupported), emit an explicit FAILING `unvalidated:` step - -never silence. For .NET: build preferred solution + independently build -uncovered csproj. - -## H. Unbounded validation time (Sol #11) -Add a total per-project validation deadline (default 45 min, constant) and a -generous root cap (e.g. 12); overflow emits explicit failed `unvalidated:` -steps. Wire into the multi-root loops. - -## I. Repeat ID stability (Sol #10) -`runner.ts:875` omits `-r01` when repeats===1. Fix: ALWAYS suffix new run IDs -with the trial number; on resume, migrate/match legacy unsuffixed IDs as trial -1 (do not regenerate them); reject resuming a dir whose recorded runProtocol -(repeats/seed) conflicts with the current invocation unless artifacts align. - -## J. Budget-exhaustion on estimates (Fable #6) -`scoring.ts:501-523` - flipping a PASSING run to budget-exhausted off a -hand-maintained price estimate is fragile. Apply post-hoc budget exhaustion -only when estimated cost exceeds cap * 1.25 (tolerance constant), and record -`budgetEstimated: true` in the outcome evidence. Test the tolerance boundary. - -## K. Timeout usage salvage accuracy (Fable #7) -`agents/claude.ts` - salvage currently takes the LAST usage-bearing message -(per-message usage → undercount). Fix: SUM per-message usage across the -partial stream. `agents/codex.ts` - summing `turn.completed` usages assumes -per-turn deltas; add a monotonic-cumulative detector: if successive usage -totals are non-decreasing supersets, take the last instead of summing. Comment -the heuristic; test both shapes. - -## L. --skip-validation mislabeled (Fable #11) -`runner.ts:304-310` - skip-validation runs persist outcome "model-failure". -They must persist an explicit unmeasured/skipped outcome that scoring treats -as not-scored (excluded from denominators), with a test. - -## M. Python import smoke safety (Fable #10) -`validation/index.ts:842-863` - first-alphabetical module can shadow an -installed package (vacuous pass) or execute arbitrary top-level code. Fix: -import the project's module via importlib from its FILE PATH (spec_from_file_ -location) so the project source is what's imported; prefer packages containing -an __init__; skip modules whose name collides with an installed distribution -only when file-path import is impossible. Keep configured mypy/pyright as the -preferred gate. Test: missing third-party import in project source must fail -validateProject end-to-end (fixture). - -## N. introducedAt is one bucket (Fable #12) -All specs say 2026-07-10 (file-split date). Backfill from when each spec's -CONTENT landed: use `git log -S '' --reverse` over the deleted -`scripts/scaffbench-v2-lib.ts` history and PR dates. Expected shape: original -5 core specs late June (v2 readiness 2026-06-30); expansion batches 1-3 in -early July (PR #282 era, before the 2026-07-06 fable5 run for the 13-spec -suite). Exact dates from git history, not guesses; document the command used -per spec in the report. - -## O. Test-quality gaps called out by Sol (implement alongside the above) -- Repeat aggregation test WITH a stepless failed trial (item A). -- Shuffle/interleave test covering cross-round adjacency + resume with changed - repeat count (item I). -- Expo test asserting install → export ORDERING in the planned steps. -- Python end-to-end missing-import failure (item M). -- .NET solution-coverage test: sln + orphan csproj → orphan built or failing - step emitted (item G). -- Nested independent bun/cargo root with broken child → child validated (G). -- Idle-timeout tests per item D. -- Prompt snapshot test per item C. -- Adversarial classification fixtures per items E/F. diff --git a/testing/scaffbench-hardening-report-2026-07-17.md b/testing/scaffbench-hardening-report-2026-07-17.md deleted file mode 100644 index 8446d9972..000000000 --- a/testing/scaffbench-hardening-report-2026-07-17.md +++ /dev/null @@ -1,153 +0,0 @@ -# ScaffBench hardening change report (2026-07-17) - -Implemented the complete `testing/scaffbench-hardening-spec-2026-07-17.md` scope. No benchmark sweep was run, no benchmark run directory was modified, and no file under `apps/web/` was changed. - -Version stamps: - -- `HARNESS_VERSION`: `2.2.0` -- `VALIDATION_CACHE_VERSION`: `4` -- `PROMPT_VERSION`: `2026-07-17` -- `MIN_RANKED_TRIALS`: `3` - -## Per-item changes - -| Item | Files touched | Behavior change | Regression test added | -| ---- | -------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- | -| 1a | `scripts/scaffbench/agents/opencode.ts`, `scoring.ts`, `runner.ts`, `types.ts` | Parses opencode error/step-finish reasons and recognizes the exact unknown + zero-usage + no-tool + no-project provider-infra signature. | `1a derives opencode terminal reasons...` | -| 1b | `types.ts`, `scoring.ts`, `runner.ts`, `summary.ts` | Persists seven fine-grained outcomes and explicitly rolls only provider/harness/validation infra into the legacy inconclusive bucket. | `1b persists fine outcomes...` | -| 1c | `validation/classification.ts`, `validation/index.ts`, `scoring.ts` | Narrowly detects transient registry/network failures, retries install-class steps once, marks recurrence validation-infra, and keeps registry 404/model-version failures model-owned. | `1c retries a transient install once...` | -| 1d | `validation/cache.ts`, `validation/classification.ts` | Rejects timeout, spawn-error, and any transient-signature failure from the validation cache; green results remain cacheable. | `1d never caches transient failures...` | -| 1e | `scoring.ts` | Treats core timeouts as model failures unless backed by transient/spawn evidence; advisory timeouts do not erase a green core verdict. | `1e scores core timeouts...` | -| 1f | `agents/command.ts`, `types.ts`, `scoring.ts` | Records normalized spawn error codes and distinguishes harness-level missing binaries from project scripts that exit 127. | `1f separates project exit 127...` | -| 2a | `agents/command.ts`, `agents/claude.ts`, `agents/codex.ts`, `agents/opencode.ts`, `runner.ts` | Always parses partial streams after termination; Claude salvages the last usage-bearing event, Codex sums observed usage events, and opencode sums observed step finishes. | `2a salvages usage from partial...` | -| 2b | `agents/command.ts`, `types.ts`, `scoring.ts`, `runner.ts` | Persists `timeout-progressing`/`timeout-stuck` from received event timestamps and recent tool/file activity; both roll up as deadline failures. | `2b tags hard timeouts as progressing...` | -| 2c | `agents/command.ts`, all generation adapters, `constants.ts` | Adds a 20-minute stdout-idle generation timeout, distinct from the hard ceiling and always tagged stuck. | `2c kills stdout-idle generations...` | -| 2d | `types.ts`, `constants.ts`, all generation adapters, `runner.ts` | Adds per-spec `timeoutMultiplier`, applies it to `GEN_TIMEOUT_MS`, and retains `CLAUDE_TIMEOUT_MS` as a compatibility alias. | `2d applies per-spec timeout scaling...` | -| 2e | `types.ts`, `runner.ts`, `scoring.ts` | Persists each run's enforced/non-enforced budget policy; only Claude receives an enforced CLI cap, while Codex/opencode overages classify post-hoc as budget exhaustion. | `2e records enforcement policy...` | -| 3a | `constants.ts`, `summary.ts`, `build-scaffbench-data.ts`, `build-scaffbench-2-1-data.ts`, `splice-scaffbench-2-1.ts` | Uses prompt weights 75/25/0 and assisted weights 60/25/15 everywhere the index is computed or emitted. | `3a uses prompt and assisted...` | -| 3b | `summary.ts` | Removes the prompt-lane discipline floor, bounding zero-validation rows to 25% of wired-libraries mean. | `3b gates a zero-validation prompt cell...` | -| 3c | `summary.ts` | Computes validation, wired libraries, discipline, faithfulness, and acceptance over the same scored trial set. | `3c computes wired and discipline...` | -| 3d | `types.ts`, `validation/index.ts`, `validation/cache.ts`, `scoring.ts`, `summary.ts`, all three board-data scripts | Persists whether quality was requested, returns `"na"` when it was not, uses a quality-only denominator, and publishes unavailable full-pass as `null`. | `3d represents an unrequested quality gate...`; existing builder test also covers null. | -| 4a | `build-scaffbench-2-1-data.ts`, `splice-scaffbench-2-1.ts` | Keys exact results by model/effort/path/spec/trial and publishes trials, scored trials, pass count/rate, pass@k, and pass^k instead of choosing an arbitrary repeat. | `4a keys every trial...` | -| 4b | `runner.ts`, `types.ts`, `summary.ts` | Moves the repeat loop outside the spec loop, seed-shuffles specs per repeat, and stamps the deterministic seed in metadata/provenance. | `4b interleaves repeat rounds...` | -| 4c | `constants.ts`, `types.ts`, `runner.ts`, `summary.ts` | Adds suite/harness/cache/prompt/adapter/trial provenance and marks only consistent rows with at least three trials per cell `ranked`; others are `exploratory`. | `4c marks only version-consistent rows...` | -| 5a | `validation/index.ts` | Prefers `.sln`/`.slnx`; without one, restores/builds every discovered `.csproj` with namespaced steps. | `5a prefers a solution...` | -| 5b | `validation/index.ts` | Keeps `compileall`, runs configured mypy/pyright, or performs a src-layout entry-module import smoke that exposes missing runtime imports. | `5b runs configured Python typecheckers...` | -| 5c | `validation/index.ts` | Upgrades Rust validation to `cargo check --workspace --all-targets`. | `5c invokes cargo check...` | -| 5d | `validation/index.ts`, `scoring.ts` | Replaces mutating `go mod tidy` gating with `go mod download` + `go build ./...`; tidy runs against a disposable copy as an advisory diff. | `5d uses go mod download...` | -| 5e | `validation/index.ts` | Detects Expo packages and runs non-interactive `npx expo export`, adding `--platform web` when web dependencies are configured; raw exit-126 diagnostics remain intact. | `5e plans a non-interactive Expo export and preserves exit-126 diagnostics` | -| 5f | `types.ts`, `validation/index.ts`, `specs/frontier-polyglot-proto.ts` | Adds ordered `prerequisiteCommands`, gates builds on them, and wires `buf generate` for the polyglot proto spec. | `5f executes spec-declared prerequisites...` | -| 5g | `validation/index.ts` | Removes the three-root cap, validates every independent manifest root, and emits a failing skip when a discovered root/project cannot be validated. | `5g validates more than three...` | -| 5h | `validation/cache.ts`, `summary.ts` | Adds platform, architecture, and toolchain-version hash to cache identity; source hashing now includes modes, directory entries, and symlink targets. | `5h includes environment/toolchains...` | -| 6a | `types.ts`, `cli.ts`, `runner.ts` | Adds opt-in `--repair`; failed measured cells re-run the same adapter in the archive with a 50-line diagnostic prompt, revalidate, and store a separate repair result without replacing pass@1. | `6a keeps repair off by default...` | -| 6b | `calibrate.ts`, `cli.ts`, `runner.ts`, `index.ts` | Adds `scaffbench calibrate --spec `, runs the fixed weak model and configured strong model once each through the runner, and prints keep/cut/inconclusive. | `6b parses calibrate...` | -| 6c | `types.ts`, every file under `specs/`, `summary.ts` | Backfills `introducedAt` from `git log --follow --diff-filter=A` (all current spec files resolve to `2026-07-10`) and renders introduction-cohort pass rates. | `6c backfills ISO introduction dates...` | -| 6d | `build-scaffbench-2-1-data.ts` | Emits a stderr discrimination row per spec with cross-model pass spread plus ceiling (>90%) and floor (0%) flags. | `6d reports per-spec model spread...` | - -## Test and typecheck results - -- `bun test scripts/` - **105 passed, 0 failed**. -- `bun x tsc --noEmit -p scripts/scaffbench/tsconfig.json` - **clean**. -- `git diff --check` - **clean**. - -The focused TypeScript config is under `scripts/scaffbench/` and uses a dependency-light declaration shim for the dynamically imported route-check helpers, avoiding package build output or changes outside the permitted scope. - -## Scope audit - -- Modified implementation files only under `scripts/scaffbench/**`, the three permitted board-data scripts, and script tests. -- Added this required report under `testing/`. -- Did not modify `apps/web/**`. -- Did not modify any `testing/llm-benchmarks/**` run directory. -- Did not run a benchmark sweep or start a development server. - -## Round 2 (2026-07-17) - -Implemented every item in `testing/scaffbench-hardening-fixes-spec-2026-07-17.md`. Round 2 keeps all implementation and regression-test changes under `scripts/**`; this report append is the only requested change outside that tree. No file under `apps/web/**` or `testing/llm-benchmarks/**` was modified, and no benchmark sweep or development server was run. - -Round-2 version stamps: - -- `HARNESS_VERSION`: `2.2.1` -- `VALIDATION_CACHE_VERSION`: `5` -- `PROMPT_VERSION`: `2026-07-17-round-2` -- Total project-validation deadline: 45 minutes -- Validation root cap: 12 -- Estimated-budget tolerance: 1.25× the configured cap - -### A–O fixes and regressions - -| Item | Files | Fix | Failure-scenario regression | -| ---- | ------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| A | `scripts/benchmarks/build-scaffbench-2-1-data.ts` | Derives scored/pass counts from each raw result's persisted outcome plus raw Core evidence. A scored stepless/no-project failure remains in the denominator. Summary `scoredRuns`/`passCount` mismatches now throw instead of being repaired with `min`. | `A counts a scored stepless failure and rejects aggregate mismatches` covers `{no project/no steps fail, pass, pass}` → `2/3`, plus corrupt aggregate rejection. | -| B | `scripts/benchmarks/splice-scaffbench-2-1.ts` | Null-propagates both quality count and rate when `fullPass` is `null`. | `B preserves null quality through splice normalization`. | -| C | `scripts/scaffbench/specs/*.ts` | Removes the contradictory dependency-install prohibition from every spec body while retaining the CLI scaffold's explicit `--no-install` policy. Self-verification installs remain allowed. | `C snapshots a non-contradictory install policy for every path mode` generates every spec under prompt/MCP/CLI modes, rejects prohibition wording, and snapshots the allowed install lines. | -| D | `scripts/scaffbench/agents/{command,agy,claude,codex,opencode}.ts`, `agents/index.ts`, `types.ts` | Makes idle enforcement an explicit adapter capability: Claude/Codex/opencode/Kilo only; agy disabled. Both stdout and stderr count as activity. JSONL tool-start/completion state suspends idle enforcement while a tool call is in flight; the hard deadline remains active. | `D treats stderr as activity and suspends idle kill during an in-flight tool` also verifies the buffered-adapter capability map and delayed buffered output. | -| E | `scripts/scaffbench/validation/classification.ts`, `scoring.ts` | Requires adjacent HTTP/status/error context for 429/5xx, package-fetch/registry context for HTTP transients, and line-distinct transient evidence when 404/version-not-found is also present. Recurrence now requires `retryCount > 0` plus `transientNetwork: true`; raw signatures remain available to cache exclusion. Provider-infra tokens are word-bounded and error-contextual. | `E rejects adversarial transient/provider false positives and requires retry evidence` covers `531 packages installed`, `0.429.1`, mixed 404+503, real 429, repeated raw lifecycle noise without retry evidence, genuine retry recurrence, capacity prose, and `14293`. | -| F | `scripts/scaffbench/agents/opencode.ts` | Tracks non-empty assistant text and refusal/moderation parts. The unknown/zero-usage/no-tool provider signature now additionally requires no assistant text. | `F keeps an opencode zero-usage refusal with assistant text model-owned`. | -| G | `scripts/scaffbench/validation/index.ts` | Parses bun/npm workspace globs (including object form, braces, exclusions), Cargo `[workspace]` members/excludes, and preferred-solution `.sln`/`.slnx` project membership. Nested manifests not actually covered by a parent are independently validated. A preferred .NET solution is built together with all uncovered `.csproj` files. Unhandled discovered roots emit failing `unvalidated:*` evidence. | `G builds a preferred .NET solution plus an uncovered orphan project` and `G validates broken nested bun and Cargo roots outside parent membership`. | -| H | `scripts/scaffbench/constants.ts`, `validation/index.ts` | Adds the 45-minute total deadline and 12-root cap across multi-root validation. Deadline and overflow paths emit explicit failing `unvalidated:*` steps. | `H caps validation roots and emits a failing step for overflow` and `H enforces a total validation deadline with explicit failure evidence`. | -| I | `scripts/scaffbench/runner.ts`, `types.ts` | New IDs always end in `-rNN`. Resume matching recognizes the legacy unsuffixed ID only as trial 1 without renaming/regenerating it. Summaries persist `{repeats, seed}` as `runProtocol`; conflicting resumes are rejected unless every existing artifact aligns with the current schedule. Round-boundary shuffles avoid same-spec adjacency when another runnable spec exists. | `I always suffixes run IDs and prevents same-spec cross-round adjacency` and `I resumes a legacy trial 1 when repeat count grows and rejects unaligned artifacts`. | -| J | `scripts/scaffbench/{constants,scoring,summary,runner,types}.ts` | Post-hoc estimated cost exhausts budget only above 1.25×. Estimated budget outcomes persist `outcomeEvidence.budgetEstimated: true`; directly reported budget terminal reasons are not mislabeled estimated. | `J tolerates estimated cost through 1.25x and persists estimated evidence above it`. | -| K | `scripts/scaffbench/agents/{claude,codex}.ts` | Claude partial accounting sums per-message usage. Codex uses the final usage snapshot when events are monotonically non-decreasing field-wise supersets; otherwise it sums per-turn deltas. | `K sums partial Claude messages and detects cumulative versus delta Codex usage`. | -| L | `scripts/scaffbench/{runner,scoring,summary,types}.ts` | `--skip-validation` persists `validation.skipped: true` and outcome `skipped`; skipped runs are excluded from scored and quality denominators and remain eligible for later validation. | `L persists skip-validation as skipped and excludes it from denominators`. | -| M | `scripts/scaffbench/validation/index.ts` | Prefers a package `__init__.py`, then loads the selected project source with `importlib.util.spec_from_file_location`, registers that exact module, and executes it from its file path. Configured mypy/pyright still take precedence. | `M imports the project package by file path so an installed-name collision cannot pass` uses a project package named `pip` whose source imports a missing dependency. | -| N | `scripts/scaffbench/specs/*.ts` | Replaces the file-split date with each spec content's first historical commit date. | `N uses content-introduction dates from git history instead of the file-split date`. | -| O | `scripts/benchmarks/scaffbench-hardening-round-2.test.ts` plus the implementation files above | Adds the requested stepless aggregation, prompt, idle, adversarial classification/refusal, workspace/Cargo/.NET, Python end-to-end, cross-round/resume, and Expo-ordering coverage. | `O executes Expo install before export in the actual validation plan` asserts `install` precedes Expo `build/export`; the other O gaps are asserted by their corresponding A, C, D, E, F, G, I, and M regressions above. | - -### `introducedAt` provenance - -Each command below was run against the deleted monolith's history. The author date of the first `-S` hit supplies the ISO day. PR history corroborates the cohorts: original ScaffBench 2.1 content merged in PR #252 on 2026-06-25, the restraint spec in PR #260 on 2026-06-25, and expansion batches 1–3 in PR #282 on 2026-06-30. - -| Spec | Date | First content commit | Command used | -| ------------------------------- | ---------- | -------------------- | ----------------------------------------------------------------------------------------------------------------- | -| `ai-search-workbench` | 2026-06-25 | `c1596178d` | `git log --all -S 'ai-search-workbench' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `rust-leptos-axum` | 2026-06-25 | `c1596178d` | `git log --all -S 'rust-leptos-axum' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `python-ingestion-api` | 2026-06-25 | `c1596178d` | `git log --all -S 'python-ingestion-api' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `go-realtime-api` | 2026-06-25 | `c1596178d` | `git log --all -S 'go-realtime-api' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `multi-dotnet-ops` | 2026-06-25 | `c1596178d` | `git log --all -S 'multi-dotnet-ops' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `ts-minimal-restraint` | 2026-06-25 | `3b083e2ed` | `git log --all -S 'ts-minimal-restraint' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `ts-svelte-edge-orpc` | 2026-06-30 | `24299a5e2` | `git log --all -S 'ts-svelte-edge-orpc' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `dotnet-blazor-cqrs` | 2026-06-30 | `24299a5e2` | `git log --all -S 'dotnet-blazor-cqrs' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `multi-ts-go-grpc` | 2026-06-30 | `24299a5e2` | `git log --all -S 'multi-ts-go-grpc' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `java-spring-jooq-keycloak` | 2026-06-30 | `877ccbaf9` | `git log --all -S 'java-spring-jooq-keycloak' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `elixir-broadway-absinthe` | 2026-06-30 | `877ccbaf9` | `git log --all -S 'elixir-broadway-absinthe' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `react-native-expo` | 2026-06-30 | `b6ce0efc1` | `git log --all -S 'react-native-expo' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `frontier-polyglot-proto` | 2026-06-30 | `b6ce0efc1` | `git log --all -S 'frontier-polyglot-proto' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | -| `frontier-effect-eventsourcing` | 2026-06-30 | `b6ce0efc1` | `git log --all -S 'frontier-effect-eventsourcing' --reverse --format='%h %aI %s' -- scripts/scaffbench-v2-lib.ts` | - -### Round-2 verification - -- `bun test scripts/benchmarks/scaffbench-hardening-round-2.test.ts` - **18 passed, 0 failed** (134 assertions). -- `bun test scripts/` - **123 passed, 0 failed** (414 assertions). -- `bunx tsc --noEmit -p scripts/scaffbench/tsconfig.json` - **clean**. -- Install-policy contradiction grep across `scripts/scaffbench/specs` and `prompts.ts` - **no matches**. -- `git diff --check` - **clean**. - -## Code-volume metric (LoC) (2026-07-18) - -Implemented all six items in `testing/scaffbench-loc-spec-2026-07-18.md`. Implementation and regression-test changes are confined to `scripts/**`; this report append is the only source change outside that tree. No `apps/web/**` file was edited or regenerated, and no benchmark sweep or development server was run. - -| Item | Files | Result | -| ---- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | `scripts/scaffbench/code-metrics.ts`, `validation/shared.ts` | Adds `measureProjectCode`, sharing the project-tree directory exclusions with validation while excluding named lockfiles, binary extensions, and NUL-sniffed files. Counts bytes and newline-based lines, including an unterminated final line. | -| 2 | `scripts/scaffbench/{runner,types,index}.ts` | Measures the generated directory during scoring, before deferred validation installs/builds, and persists optional per-result `{files, lines, bytes}` metrics. | -| 3 | `scripts/scaffbench/summary.ts` | Adds scored-trial `avgLines` to per-spec cells and macro-averages scored cells for leaderboard rows; unavailable metrics remain `null`. | -| 4 | `scripts/benchmarks/build-scaffbench-2-1-data.ts`, `scripts/benchmarks/splice-scaffbench-2-1.ts`, `scripts/benchmarks/build-scaffbench-2-2-data.ts` via the shared publisher | Publishes `lines`, falls back to scored raw result metrics for legacy summaries, and preserves `null` for existing cells with no LoC evidence. | -| 5 | `scripts/benchmarks/backfill-scaffbench-code-metrics.ts` | Adds an idempotent, `--force`-aware backfill with repository-relative defaults and recomputes per-spec `avgLines`. The comment records that already-pruned archives make dependency/build-tree exclusions a no-op difference. | -| 6 | `scripts/benchmarks/scaffbench-code-metrics.test.ts`, `scripts/benchmarks/scaffbench-hardening.test.ts` | Covers named lockfiles, extension and sniffed binaries, nested excluded directories, unterminated lines, scored-only/macro aggregation, legacy publisher null propagation and raw fallback, and backfill idempotency. | - -### Cohort backfill - -Ran the backfill against the three required 2.2 cohort directories. Each summary now has `codeMetrics` on all 39 results and `avgLines` on all 13 per-spec cells: - -- Sol: 39 projects measured; 13 cells aggregated. -- Terra: 39 projects measured; 13 cells aggregated. -- Luna: 39 projects measured; 13 cells aggregated. -- Immediate second run: 0 projects measured and all three summaries unchanged. - -The web 2.2 data generator was intentionally not run; the operator remains responsible for regenerating that file. - -### LoC verification - -- `bun test scripts/benchmarks/scaffbench-code-metrics.test.ts` - **4 passed, 0 failed** (12 assertions). -- `bun x tsc --noEmit -p scripts/scaffbench/tsconfig.json` - **clean**. -- `cd scripts && bun test .` - **127 passed, 0 failed** (426 assertions across 7 files). diff --git a/testing/scaffbench-hardening-spec-2026-07-17.md b/testing/scaffbench-hardening-spec-2026-07-17.md deleted file mode 100644 index 820403a6b..000000000 --- a/testing/scaffbench-hardening-spec-2026-07-17.md +++ /dev/null @@ -1,192 +0,0 @@ -# ScaffBench hardening - implementation spec (2026-07-17) - -Synthesized from two independent research reports (Fable 5 + GPT-5.6 Sol) over -`scripts/scaffbench/`, the 2026-07-10 validator audit, and reference practice -from SWE-bench Verified / Terminal-Bench / Aider / LiveCodeBench. - -SCOPE RULES - -- You may modify: `scripts/scaffbench/**`, `scripts/benchmarks/build-scaffbench-data.ts`, - `scripts/benchmarks/build-scaffbench-2-1-data.ts`, `scripts/benchmarks/splice-scaffbench-2-1.ts`, - and test files under `scripts/`. -- Do NOT modify anything under `apps/web/` (a parallel work stream owns it). -- Do NOT run benchmark sweeps. Do NOT touch `testing/llm-benchmarks/` run dirs - (a live sweep is writing there). -- Acceptance: `bun x tsc --noEmit -p .` clean for the touched tsconfig scope (or - the repo's existing check for scripts), `bun test scripts/` (existing harness - tests, e.g. scaffbench-v2-lib.test.ts) green, plus NEW regression tests for - each numbered item below. Write a change report to - `testing/scaffbench-hardening-report-2026-07-17.md` listing per item: files - touched, behavior change, test added. -- Bump `HARNESS_VERSION` to "2.2.0" and `VALIDATION_CACHE_VERSION` to 4 once, - with comments summarizing this batch. - -## 1. Automatic infra-vs-model classification (replaces hand-purging) - -1a. `agents/opencode.ts` - `parseOpencodeResult` currently hardcodes -`terminal_reason: undefined`. Derive it from the JSONL stream: error events, -`step_finish` with `reason` values, zero-usage detection. A run with zero output -tokens, no tool events, and no project directory must classify as -`provider-infra`, never model-failure. opencode masks provider 429s by retrying -internally then emitting `reason:"unknown"` with zero tokens - detect exactly -that signature. - -1b. `scoring.ts` - replace the binary infra-inconclusive logic with evidence- -backed outcome categories: `success | model-failure | provider-infra | -harness-infra | validation-infra | budget-exhausted | deadline-exhausted`. -Preserve the current three-way rollup for aggregate compatibility -(provider/harness/validation-infra all count as inconclusive), but persist the -fine-grained category on the run result for the board/notes. - -1c. Transient-network signatures: in the validation step classifier, match -install-class step stderr/stdout tails against a narrow list (EAI_AGAIN, -ENOTFOUND, ETIMEDOUT, ECONNRESET, HTTP 429, 5xx from registries, TLS -handshake). On first match retry that step once (`validation/index.ts`); if it -recurs → `validation-infra`. Registry 404 / nonexistent package version is a -MODEL failure, not infra - keep that distinction explicit and tested. - -1d. `validation/cache.ts` (`cacheableValidation`) - never cache a failure whose -steps match a transient signature; keep caching timeouts/spawnErrors excluded -as today; cache green results always. - -1e. Fix the every-timeout-is-infra hole (audit F4): a CORE-step timeout is a -model failure unless a transient signature or spawn-level evidence indicates -infra (a model-authored watch-mode build must not erase the run from the -denominator). An ADVISORY-step timeout must not invalidate an already-measured -core verdict. - -1f. Distinguish exit-127-from-project-script (model failure: their script -references a missing binary) from harness-level missing toolchain -(harness-infra). Record spawn error codes on steps. - -## 2. Timeout & accounting - -2a. `agents/command.ts` + each `parse*Result` - on `timedOut`, salvage token/ -cost accounting from the partial event stream (walk the same JSONL events; for -codex sum `turn.completed`/usage events seen so far; for opencode sum -`step-finish` parts; for claude use the last result-bearing event). The result -must carry usage even when the process was SIGTERMed. - -2b. Tag timed-out runs `timeout-progressing` vs `timeout-stuck` from event -timestamps: tool/file activity within the last 10 minutes of the window → -progressing. Both score as failures; persist the tag on the result. - -2c. Idle timeout: kill a generation after 20 minutes with NO stream activity -(no stdout bytes), classified `timeout-stuck`, separate from the hard ceiling. -Implement in the command runner via last-activity tracking. - -2d. Per-spec hard-ceiling scaling: optional `timeoutMultiplier?: number` on -`BenchmarkSpec` (default 1) applied to `CLAUDE_TIMEOUT_MS` in the runner (rename -that constant to `GEN_TIMEOUT_MS`; keep an alias export if referenced widely). - -2e. Budget normalization: only the Claude adapter receives `maxBudgetUsd`. -Record in run metadata, per run: the budget policy actually in force -(`budgetEnforced: boolean`, value). Where codex/opencode report cumulative cost -in-stream, detect budget exhaustion post-hoc (cost > cap) and classify -`budget-exhausted` instead of silently letting it ride. - -## 3. Scoring & index - -3a. `constants.ts` `SCAFFBENCH_INDEX_WEIGHTS` - make path-dependent: prompt -lane {validation: 0.75, wiredLibs: 0.25, discipline: 0} (discipline is 100% in -all 317 published cells - a constant, not a signal); assisted lanes keep -{0.6, 0.25, 0.15}. Mirror wherever weights are consumed (`summary.ts`, -`build-scaffbench-2-1-data.ts` W, splice script W). - -3b. Gate the composite: a row with zero validation passes cannot exceed index -= 0.25 \* wired mean (i.e. no discipline floor). Simply follows from 3a for the -prompt lane; assert with a test. - -3c. Consistency: validation aggregates exclude infra-inconclusive runs but -wired/discipline means currently average over ALL runs (`summary.ts`). Compute -every component over the same eligible (scored) trial set. - -3d. Quality-tier honesty: `qualityPassed` returns vacuous true when the quality -gate never ran. Persist `qualityGateRequested` in `ProjectValidation`; when not -requested, quality is `"na"`, not pass. `fullPass` consumers -(build-scaffbench-data.ts) treat "na" as null/unavailable, not false and not -true. - -## 4. Trial integrity (prerequisite for repeats) - -4a. `build-scaffbench-2-1-data.ts` and `splice-scaffbench-2-1.ts` key results -by `path|specId` only - a multi-trial dir silently publishes an arbitrary -trial. Key by `model|effort|path|spec|trial`; when trials > 1, derive cell -verdicts from aggregate pass counts (pass rate, pass@k, pass^k are already -computed in `summary.ts`), and emit per-cell `trials: number`. - -4b. `runner.ts` - interleave repeats: trial loop OUTSIDE the spec loop so trial -2 of spec A doesn't immediately follow trial 1 (temporal decorrelation). -Randomize spec order within a trial with a seeded shuffle; record the seed in -metadata. - -4c. `summary.ts` / metadata: add `publicationEligibility`: a row is -`"ranked"`-eligible when all its cells share suite version, harness version, -validator cache version, prompt version (add a PROMPT_VERSION constant), agent -adapter, and trials >= a `MIN_RANKED_TRIALS` constant (set 3; single-trial -sweeps mark `"exploratory"`). This is metadata only - no behavior change to -sweeps. - -## 5. Validator v4 (verdict-changing; VALIDATION_CACHE_VERSION → 4) - -5a. .NET: prefer building the `.sln`/`.slnx` when present; otherwise build -EVERY discovered `.csproj` root (namespaced steps machinery exists). Remove the -current build-only-`apps/server`-or-roots[0] behavior. - -5b. Python: if the project configures a typechecker (mypy.ini / pyright section -in pyproject) run it; else run an entry-module import smoke -(`python -c "import "` against the src layout); keep `compileall` as an -additional syntax gate. Missing third-party imports must fail. - -5c. Rust: `cargo check --workspace --all-targets` minimum (current: bare -`cargo check`). - -5d. Go: replace mutating `go mod tidy` gate with `go mod download` + -`go build ./...`; run tidy as ADVISORY diff (report, don't gate). - -5e. Expo/React Native: add non-interactive `npx expo export` (or -`expo export --platform web` when web configured) as the build step; exit-126 -class failures should surface the actual command error. - -5f. Spec-declared codegen prerequisites: `prerequisiteCommands?: string[][]` on -`BenchmarkSpec`, run in order before component builds (e.g. `buf generate`, -`sqlc generate`). Wire frontier-polyglot-proto's proto codegen as the first -user. - -5g. Remove/raise the 3-root validation cap; an unvalidated root must surface as -its own failed/na step, never silently allow a pass. - -5h. Cache identity: include platform, arch, and `collectToolchainVersions()` -output hash in `validationCacheKey`; include file modes and symlink targets in -`hashProjectSource`. - -## 6. Batch-4 features (behind flags; no default behavior change) - -6a. `--repair` flag: after `validatePendingResults`, for each failed cell -re-invoke the SAME agent in the archived project dir with the failing step's -stderr tail (~50 lines) appended to a short repair prompt; re-validate; store -as `repair` result alongside pass@1 (new field, not a new path). Skipped -entirely without the flag. - -6b. `scaffbench calibrate --spec `: runs the spec on a weak model -(opencode/deepseek-v4-flash-free) and a strong model (config default) once -each, prints keep/cut per the weak-fails/strong-passes rule from constants.ts -comments. New subcommand file; reuse runner machinery. - -6c. `introducedAt: string` (ISO date) on every `BenchmarkSpec` (backfill: -existing 13 specs use their git introduction dates - find via `git log ---follow --diff-filter=A -- `); surface per-cohort pass rates in -`summary.md` output. - -6d. Discrimination report: in `build-scaffbench-2-1-data.ts`, emit (stderr) a -per-spec pass spread across models; flag ceiling (>90% pass) and floor (0%) -specs. - -## Explicitly OUT of scope - -- Registry snapshotting/containerized validation (live registries are the - bench's subject matter - drift is managed by 5h + run-metadata stamps). -- Retrofitting functional verifiers/oracles onto all existing specs. -- Any `apps/web` change. -- Re-scoring the published board (that's a separate operator action after this - lands). diff --git a/testing/scaffbench-loc-spec-2026-07-18.md b/testing/scaffbench-loc-spec-2026-07-18.md deleted file mode 100644 index 84b2f352d..000000000 --- a/testing/scaffbench-loc-spec-2026-07-18.md +++ /dev/null @@ -1,76 +0,0 @@ -# ScaffBench: code-volume metric (LoC) - implementation spec - -Motivation: two rows can both pass a spec while one wrote 1k lines and the -other 10k. Code volume is a visible dimension (like tokens/cost/steps), NOT -part of the index. - -SCOPE: `scripts/**` only (harness + publishers + a backfill script + tests). -Do NOT touch `apps/web/**` - the web consumer is being built in parallel and -will read the exact field shapes defined here. No benchmark sweeps. -Acceptance: tsc scripts scope clean; `cd scripts && bun test .` green -(root-cwd bun test loses child pipes on this machine - always run from -scripts/); new regression tests per item. Append a section to -testing/scaffbench-hardening-report-2026-07-17.md. End with exactly: LOC COMPLETE - -## 1. measureProjectCode(dir) - new module scripts/scaffbench/code-metrics.ts - -Walk the generated project (the ARCHIVED generation output, pre-validation): - -- Skip directories: node_modules, target, .git, deps, \_build, vendor, Pods, - .venv, .dart_tool, .gradle, obj, bin, dist, build, .next, .expo, coverage - (reuse/share the validation walk skip-list where practical). -- Skip machine-generated lockfiles by NAME: bun.lock, bun.lockb, - package-lock.json, yarn.lock, pnpm-lock.yaml, Cargo.lock, go.sum, mix.lock, - poetry.lock, uv.lock, Pipfile.lock, packages.lock.json, composer.lock, - Gemfile.lock, gradle.lockfile. -- Skip binaries: any file whose first 8KB contains a NUL byte; also by - extension (png,jpg,jpeg,gif,webp,ico,pdf,woff,woff2,ttf,otf,eot,zip,jar, - wasm,keystore,p8,p12,db,sqlite). -- Count every remaining file: lines (newline count, +1 for unterminated last - line of non-empty files), bytes. - Return { files: number, lines: number, bytes: number }. - -## 2. Persist at scoring time - -In the runner where scoreProject runs on generatedDir (generation phase, -BEFORE validation installs anything), compute measureProjectCode and persist -on the run result as `codeMetrics: { files, lines, bytes }` (null/absent when -no project dir). Add to types.ts. - -## 3. Aggregates - -summary.ts bySpecCell: `avgLines` (mean over SCORED trials, same eligibility -set as other metrics; null when nothing scored). Leaderboard rows: `avgLines` -mean over scored cells. - -## 4. Publishers - -- build-scaffbench-2-1-data.ts PublishedCell: add `lines: number | null` - - from cell aggregate avgLines when present, else recompute mean from raw - results' codeMetrics, else null (legacy summaries). -- splice normalization: null-propagate (existing cells without the field - stay null; never fabricate). -- build-scaffbench-2-2-data.ts cells: emit `lines` the same way. - -## 5. Backfill script scripts/benchmarks/backfill-scaffbench-code-metrics.ts - -For each dir passed as argv (default: the three 2.2 cohort dirs under -testing/llm-benchmarks/v2-codex-{sol,terra,luna}/gpt-5-6-\*-high-r3-2026-07-17): -for every result with a projectDir that exists on disk, compute -measureProjectCode and write codeMetrics into summary.json results (idempotent; -skip results that already have codeMetrics unless --force). Recompute the -bySpecCell avgLines aggregates consistently. NOTE: archived projects were -PRUNED of node_modules/target etc. - the skip list makes this a no-op -difference, which is the point; state this in a comment. -Then RUN the backfill on the three cohort dirs and regenerate -apps/web/src/components/home/scaffbench-2-2-data.ts is FORBIDDEN (web scope) - -instead just run the backfill so summaries carry codeMetrics; the operator -regenerates the web data file. - -## 6. Tests (scripts/, assert failure scenarios) - -- lockfiles and binaries excluded; nested skip dirs excluded; unterminated - last line counted. -- cell avgLines uses scored trials only. -- publisher null-propagation for legacy cells without the field. -- backfill idempotency. diff --git a/testing/scaffbench-validation-footprint-investigation-2026-08-21.md b/testing/scaffbench-validation-footprint-investigation-2026-08-21.md deleted file mode 100644 index 22f27b4d9..000000000 --- a/testing/scaffbench-validation-footprint-investigation-2026-08-21.md +++ /dev/null @@ -1,419 +0,0 @@ -# ScaffBench validation footprint investigation (2026-08-21) - -Author: gpt-5.6-sol (xhigh reasoning, read-only codex session 01a02404), commissioned to find lower-machine-footprint verification approaches. Claims below were spot-verified against the harness by the primary agent; see scaffbench-benchmark.md for the protocol this must respect. - -ScaffBench is serial only at the project level. It does not constrain the parallelism inside Cargo, Go, Bun, uv, Gradle, Erlang, test runners, or model-authored build scripts. That explains why concurrency 1 can still saturate a 10-core Mac. - -The best first move is a documented `low-v7` validation resource profile with two-worker tool caps, macOS background scheduling, longer timeouts, fail-fast execution, and reliable process-tree cleanup. This should cut peak CPU and memory without weakening Core or Full verdicts. A containerized or remote validation worker is the stronger long-term answer because the current host execution is not a security boundary. - -No useful CPU, peak-RSS, or energy telemetry exists in the current results. The percentage estimates below refer to reductions in documented concurrency limits, not measured reductions on this laptop. - -## What the harness does today - -- [validatePendingResults()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/runner.ts:741) validates projects with `concurrency: 1`, then [validateProject()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:276) runs discovered ecosystems sequentially. -- Each ecosystem can still use every logical CPU. [commandStep()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:38) passes no resource limits. -- [runCommand()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/agents/command.ts:43) gives children the complete harness environment, collects their complete output in memory, and kills only the direct child on timeout. -- The current limits are 10 minutes per step and 45 minutes per project in [constants.ts](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/constants.ts:74). Throttling without increasing these limits would create false timeout failures. -- [validateProjectCached()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/cache.ts:31) hashes the archive before validation. Validation then mutates that archive through lockfiles and generated output. A forced revalidation may therefore hash a different tree and miss the cache. -- The cache lives under one output directory. It does not reuse an identical validation from another run directory. -- [collectToolchainVersions()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/summary.ts:530) starts eight version probes for every cache lookup. It omits Bun, Node, Java, Maven, Gradle, Elixir, Erlang, and Buf. -- Root discovery repeatedly walks the same source tree. Bun, Cargo, Python, Go, .NET, Java, and Elixir each perform one or more independent scans. -- The 13-spec cohort contains seven Bun projects, three Go validations, two Cargo validations, two .NET validations, and one each for Python, Java, and Elixir. Caps on Bun, Go, Cargo, and .NET cover most of the cohort. - -## Recommended low-resource profile - -The profile must be fixed for published rows and included in provenance and the validation cache key. Exploratory overrides are fine, but rows with different profiles should not be compared. - -| Tool | Proposed settings | Expected peak effect | Fidelity notes | -| ---------------------- | ----------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------: | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Universal macOS policy | Run validation children under `taskpolicy -b -d throttle -c background` | Lower CPU priority and throttled disk and new network I/O | `taskpolicy` policies are inherited by children. It lowers interference but is not a CPU quota. [Darwin taskpolicy manual](https://keith.github.io/xcode-man-pages/taskpolicy.8.html) | -| Bun install | `--concurrent-scripts=2 --network-concurrency=8`; `--no-save` when no lock; `--frozen-lockfile` when a lock exists | Lifecycle concurrency falls from the installed Bun default of 5 to 2. Network requests fall from 48 to 8 | Do not use `--ignore-scripts`; it changes install semantics. Bun already uses a global cache and copy-on-write cloning on macOS. [Bun install documentation](https://bun.sh/docs/pm/cli/install) | -| Node builds | Universal background policy first; cap known task runners only when the command can be identified safely | Depends on framework | `UV_THREADPOOL_SIZE` does not cap worker threads created by bundlers. A generic Node CPU cap is not reliable. Avoid a low `--max-old-space-size` until RSS data exists. | -| Cargo | `CARGO_BUILD_JOBS=2`, plus `-j 2` on check, clippy, and test | Compiler-process ceiling falls from 10 to 2, an 80% concurrency reduction | Cargo documents logical CPUs as the default. Keep `target/` project-local. [Cargo configuration](https://doc.rust-lang.org/cargo/reference/config.html) | -| Go | `GOMAXPROCS=2`; `-p=2` on build, vet, and test; `-parallel=2` on test | Package-build concurrency falls from 10 to 2. Parallel tests also fall to 2 | `-p` controls simultaneous build programs, while `GOMAXPROCS` also constrains Go runtime work. [Go command reference](https://pkg.go.dev/cmd/go) | -| .NET | `-m:1 -lowPriority -nr:false -p:BuildInParallel=false`; tests with `-p:TestTfmsInParallel=false` | One MSBuild project process, low priority, no lingering worker nodes | MSBuild already defaults to one process unless `-m` is supplied, so the main gains are explicitness, priority, and process cleanup. [MSBuild switches](https://learn.microsoft.com/en-us/visualstudio/msbuild/msbuild-command-line-reference?view=visualstudio) | -| Gradle | `--max-workers=2 --no-parallel --priority=low --no-daemon --no-watch-fs` | Worker ceiling falls from 10 to 2. No daemon holds memory after validation | Gradle defaults to processor count and leaves daemons alive by default. [Gradle CLI](https://docs.gradle.org/current/userguide/command_line_interface.html) | -| Maven | Keep single-threaded build; set artifact download threads to 2; pilot `JAVA_TOOL_OPTIONS=-XX:ActiveProcessorCount=2` | Small CPU change, lower download and JVM thread bursts | Maven is already single-threaded without `-T`. Do not impose a small JVM heap before measuring legitimate builds. [Maven configuration guide](https://maven.apache.org/guides/mini/guide-configuring-maven.html) | -| uv | `UV_CONCURRENT_BUILDS=2`, `UV_CONCURRENT_INSTALLS=2`, `UV_CONCURRENT_DOWNLOADS=8`; use `uv run --no-sync` after the first successful sync | Builds and installs fall from 10 workers to 2. Downloads fall from 50 to 8 | uv normally checks and syncs the environment on every `uv run`. `--no-sync` is equivalent after a successful `uv sync`. [uv settings](https://docs.astral.sh/uv/reference/settings/), [uv locking and syncing](https://docs.astral.sh/uv/concepts/projects/sync/) | -| Elixir and Erlang | `ERL_FLAGS="+S 2:2 +SDcpu 1:1 +SDio 1"` | Normal BEAM schedulers fall from about 10 to 2 | Confirm flags against the installed OTP version during preflight. [Erlang runtime flags](https://www.erlang.org/docs/25/man/erl.html) | - -Start with two workers. One worker minimizes footprint further but can make framework builds disproportionately slow and increase timeout risk. - -## Quick wins - -### 1. Add the fixed resource profile - -What changes: - -- Add a `ValidationResourceProfile` in [types.ts](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/types.ts:295). -- Put the published profile and revised timeouts in [constants.ts](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/constants.ts:74). -- Make [commandStep()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:38) merge the profile with step-specific environment variables. -- Apply the ecosystem settings in `validateBunProject`, `validateCargoProject`, `validatePythonProject`, `validateGoProject`, `validateDotnetProject`, `validateJavaProject`, and `validateElixirProject`. -- Record `resourceProfileId`, worker limit, QoS policy, timeout policy, and cache mode in `ProjectValidation`, summary metadata, and [validationCacheKey()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/cache.ts:105). -- Double the first pilot limits to about 20 minutes per step and 90 minutes per project. Adjust from measured data before publishing. - -Expected saving: - -- Up to 80% fewer concurrent Cargo, Go, Gradle, uv, and Erlang workers on this 10-core machine. -- 60% fewer Bun lifecycle-script jobs with a 5-to-2 cap. -- Lower disk and network interference from the Darwin background policy. -- Node framework builds remain the largest uncapped case. - -Verdict and comparability risk: - -- Low if timeouts rise with the caps. -- Medium if memory limits are added without measurements. -- Every row must use the same profile. A profile change requires a cache-version bump. - -Effort: 2 to 4 engineering days, including focused tests and documentation. - -### 2. Stop after the verdict is already determined - -The validators currently continue after a failed Bun build or typecheck, and quality gates continue after a failure or disqualifying skip. That spends CPU on commands that cannot change either tier. - -What changes: - -- Build a complete validation plan before execution in [validateProject()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:276). -- Once any definitive Core step fails, stop all later expensive steps. Core and Full are already false. -- Once Core passes and one applicable quality gate fails or skips, stop later quality commands. Full is already false. -- Put format checks before lint and tests where possible. Tests stay last. -- Detect missing Bun linter or formatter before running any quality command. Record all applicable skips, then stop. -- Add `status: "not-run"` with a reason such as `earlier-core-failure`. Update [validationPassed()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/scoring.ts:461) and `qualityPassed()` so a planned but unexecuted applicable gate can never become a pass. - -Expected saving: - -- High on weak-model runs and Full failures. -- A failed Bun build can avoid typecheck, lint, format, tests, and later native compiles. -- A format failure can avoid lint and tests. -- Savings approach all remaining validation work after the first failure. Green projects receive little benefit. - -Verdict and comparability risk: - -- Low for boolean verdicts if the plan records every applicable gate before execution. -- Diagnostic coverage decreases. Repair remains safe because it already selects the first Core failure and revalidates after the source changes. -- Do not omit planned steps silently. - -Effort: 2 to 3 days. - -### 3. Kill process groups and bound output memory - -The current timeout sends signals to the direct child. Watchers and grandchildren may survive. The repository already has a detached process-group implementation in [generated-project-proof.ts](/Users/ibrahime/Documents/Better-Fullstack/testing/generated-project-proof.ts:48). - -What changes: - -- Give validation commands their own process group in [runCommand()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/agents/command.ts:43), or introduce a validation-only executor. -- On step timeout, project deadline, interruption, or harness exit, send `SIGTERM` to the group, wait briefly, then send `SIGKILL`. -- Ensure the project-deadline finalizer kills the active group before returning `unvalidated:deadline`. -- Replace unbounded `Stream.runFold` output accumulation with bounded tails plus a streamed `validate.log`. -- Set an output ceiling, such as 16 MiB per stream. Exceeding it should terminate the group and produce explicit model-owned failure evidence. - -Expected saving: - -- Little change on normal runs. -- Bounded harness RAM regardless of command output. -- Runaway watcher CPU, RAM, and disk use ends at the timeout instead of continuing indefinitely. -- This directly fixes the known watcher-shaped process problem. - -Verdict and comparability risk: - -- Low. Timeout semantics already count Core hangs as model failures. -- The output limit becomes part of the protocol and cache identity. -- Generation commands still need full JSON output, so bounded capture should apply only to validation. - -Effort: 2 to 4 days in TypeScript. A Rust supervisor is not needed for the first version. - -### 4. Validate an immutable copy - -The archive is both the evidence artifact and the mutable validation workspace. Bun, Cargo, uv, Mix, code generation, and framework builds can create lockfiles and output directories. The source hash therefore does not remain stable. - -[archiveProjectSource()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:112) also removes every directory named `build`, `dist`, `bin`, or `obj` at any depth. A model may legitimately place source in one of those directories. - -What changes: - -- Preserve a faithful source archive. -- Create an APFS copy-on-write validation directory per project. Recursively clone files with `COPYFILE_FICLONE`, preserve modes and symlinks, and delete the clone after validation. -- Hash the pristine archive once. -- Run all prerequisites and validators against the clone. -- Use `bun install --no-save` when no Bun lock exists and frozen install when one exists. -- Replace [runGoTidyAdvisory()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:1120) with `go mod tidy -diff`. The installed Go tool documents this as non-mutating and nonzero when changes are required. - -Expected saving: - -- Reliable cache hits after forced revalidation. -- No persistent `node_modules`, `.venv`, `target`, `.output`, `_build`, or lockfile growth in archived evidence. -- `go mod tidy -diff` removes a recursive copy. -- The copy-on-write clone adds small metadata cost on the first validation but copies data only when a command changes it. - -Verdict and comparability risk: - -- Low. This improves artifact fidelity. -- Some builds inspect absolute paths, but validation already runs from an archived path rather than the generation path. -- Broad archive exclusions should be removed only after archive-size and file-count guards exist. - -Effort: 2 to 4 days. - -### 5. Remove repeated work with exact equivalence - -What changes: - -- Python: after `uv sync --all-extras`, use `uv run --no-sync` for compile, typecheck, Ruff, and pytest in [validatePythonProject()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:602). -- Go: use `go mod tidy -diff`. -- Cargo Full validation: run `cargo clippy --workspace --all-targets -- -D warnings` first. If it passes, that command can satisfy both `cargoCheck` and `lint`. If it fails, run `cargo check --workspace --all-targets` to determine Core independently. -- .NET, Maven, and Gradle can use the same optimistic-superset pattern. A passing test command includes compilation and can satisfy build plus test. If it fails or times out, run the build-only command to establish Core. -- TypeScript: reuse a build result as typecheck only when the build command explicitly and unconditionally invokes the same typecheck command. Do not infer equivalence from framework names. - -Expected saving: - -- `uv run --no-sync` avoids up to five repeated environment scans. -- Passing Cargo Full projects avoid one compiler pass. -- Passing Java and .NET projects avoid one build-tool startup and configuration phase. Their existing incremental output already limits duplicate compilation, so this gain is moderate. -- Failure cases perform the same number of heavy commands as today. - -Verdict and comparability risk: - -- Low for uv and `go mod tidy -diff`. -- Medium for superset-command reuse. Each equivalence needs a test proving that a green superset establishes both gates and that a failed advisory command still triggers the independent Core command. -- Record one physical command with multiple gate claims instead of fabricating two executions. - -Effort: 1 day for uv and Go. Another 3 to 5 days for proven superset planning. - -### 6. Improve validation caching - -What changes: - -- Memoize host toolchain probes once per harness run instead of starting eight probes for each project. -- Probe only toolchains relevant to the discovered plan. -- Add Bun, Node, Java, Maven, Gradle, Elixir, Erlang, and Buf. -- Resolve project-sensitive versions from the project directory. This matters for `global.json`, wrappers, and local tool selection. -- Move deterministic validation results to a shared content-addressed cache outside an individual output directory. -- Include the resource profile, sandbox image digest, offline mode, command-plan digest, platform, architecture, and toolchain fingerprints. -- Never cache timeouts, process-limit kills, output-limit kills, transient network failures, or memory-pressure failures. - -Expected saving: - -- A true cache hit avoids all installation and compilation. -- Memoization reduces as many as 104 small toolchain subprocesses across a 13-project run to one relevant set. -- Cross-run hits will be uncommon for independent model output, but common during repeated revalidation of canonical or unchanged archives. - -Verdict and comparability risk: - -- Low if the key is complete. -- A shared cache with an incomplete identity can create false passes across toolchain changes. -- Do not add root-level step caching yet. Untrusted build scripts can read files outside their manifest root, so a safe per-root input closure is difficult to prove. - -Effort: 2 to 4 days. - -### 7. Scan the source tree once and enforce generous artifact budgets - -What changes: - -- Replace repeated [walk()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/shared.ts:38) calls with one immutable manifest inventory. -- Use that inventory for Bun, Cargo, Python, Go, .NET, Java, and Elixir root planning. -- Reuse it during hashing where possible. -- Add documented limits for source files, source bytes, manifest roots, individual file size, and output bytes. -- An exceeded model-output limit should create explicit failure evidence. A harness inability to inspect an otherwise in-limit tree remains infrastructure. - -Expected saving: - -- Several full filesystem walks become one. -- Small normal-case improvement. -- Large protection against a generated tree containing millions of files or a huge sparse file. - -Verdict and comparability risk: - -- Low with generous limits and explicit failure records. -- The limits become benchmark rules and must appear in protocol provenance. - -Effort: 2 to 3 days. - -### 8. Use the existing two-phase mode operationally - -This needs no code. - -- Generate with `--generate-only`. -- Run the same command and output directory later with `--validate-existing`, keeping the original model, effort, spec, path, and quality arguments. -- Wrap only the validation phase with `taskpolicy`. -- Run it overnight or while the laptop is otherwise idle. -- Add a fixed 15 to 30 second cooldown between projects. This lowers average power and thermal accumulation, though it does not lower peak use during a command. -- Do not rely only on the current [validationPriority()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/runner.ts:805). It places Rust and .NET at the hot end of the queue. - -Expected saving: - -- Lower interference with interactive use. -- A fixed cooldown reduces sustained temperature and the chance that later trials run under thermal throttling. -- Total compute and energy change little. - -Verdict and comparability risk: - -- Very low. The validation commands remain unchanged. -- Fixed cooldowns are easier to document than adaptive thermal pauses. -- Adaptive waits based on load or temperature are useful operationally, but the harness should record when and why it waited. - -Effort: none for two-phase operation. Less than one day for fixed cooldown and logging. - -## Package cache strategy - -Most package managers already share download caches across projects. Simply assigning cache directories will not produce the gains of a new cache on a machine whose normal caches are warm. - -- Bun already uses `~/.bun/install/cache` and macOS clonefile installation. [Bun documentation](https://bun.sh/docs/pm/cli/install) -- Cargo uses `$CARGO_HOME` as a download and source cache. [Cargo Home](https://doc.rust-lang.org/cargo/guide/cargo-home.html) -- Go shares its module cache across projects and authenticates cached modules against `go.sum`. [Go module cache](https://go.dev/ref/mod) -- uv uses an aggressive, append-only cache and copy-on-write installation on macOS. [uv cache documentation](https://docs.astral.sh/uv/concepts/cache/) -- NuGet checks its global package folder before network sources. [NuGet cache documentation](https://learn.microsoft.com/en-us/nuget/consume-packages/managing-the-global-packages-and-cache-folders) -- Maven’s local repository defaults to `~/.m2/repository`. [Maven settings](https://maven.apache.org/settings.html) - -The useful change is isolation and repeatability: - -1. Give ScaffBench dedicated, versioned dependency caches. -2. Record cache mode and cache generation in provenance. -3. Keep build outputs project-local. Do not share Cargo `target`, Gradle build outputs, Turbo task outputs, `.venv`, or `node_modules` across untrusted trials. -4. Do not expose writable shared caches to model-authored code in a future sandbox. Use a read-only warmed snapshot plus a disposable per-project overlay. - -A pull-through mirror is a larger project. It helps cold cohorts and registry outages, but it is less important than worker caps because current package managers already avoid most repeated downloads. - -## Security finding - -The current validator executes arbitrary code on the personal host account: - -- Bun root lifecycle scripts and build scripts. -- Cargo `build.rs`. -- Python build backends and the import smoke test. -- Maven and Gradle plugins, including model-created wrapper scripts. -- `mix.exs`. -- Generated test suites and the `buf generate` prerequisite. - -[runCommand()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/agents/command.ts:103) also passes the complete harness environment. Removing tokens from the environment helps, but a process running under the same macOS user can still read files through absolute paths. - -Immediate interim controls: - -- Use a validation-specific environment allowlist. -- Remove agent credentials, cloud tokens, SSH agent variables, package publication tokens, and Git credential variables. -- Set `GIT_TERMINAL_PROMPT=0` and disable interactive package-manager prompts. -- Use a dedicated temporary `HOME`. -- Prefer a dedicated macOS account if host validation continues. - -These steps reduce accidental exposure but do not create isolation. - -## Larger projects - -### A. Containerized validation worker - -Build a pinned ARM64 Linux image with Bun, Rust, Go, .NET, Java, Elixir, uv, and Buf. Run one disposable container per project with about two CPUs, a measured memory limit, a PID limit, a read-only source mount, and a writable copy-on-write workspace. - -Expected saving: - -- A real CPU quota prevents saturation. -- A memory limit and PID limit contain runaways. -- Killing the container kills the complete process tree. -- Dedicated cache snapshots remove repeated downloads. -- Docker supports hard memory limits and CPU quotas such as `--cpus=2`. [Docker resource constraints](https://docs.docker.com/engine/containers/resource_constraints/) - -Risk: - -- Docker Desktop or another Linux VM has its own idle RAM and CPU cost. -- Linux verdicts are not macOS verdicts. -- All published 3.0 rows must use the same image digest. -- Writable shared caches would create a cross-trial poisoning channel. - -Effort: 2 to 4 weeks, including the image, cache overlays, provenance, and canonical calibration. - -This is the best same-machine security and hard-cap option, but it is not the smallest implementation. - -### B. Dedicated remote validation worker - -Run the same pinned image on a small two-core remote machine. Upload source archives and download signed validation results. - -Expected saving: - -- Nearly zero MacBook validation footprint. -- Stable CPU and memory allocation. -- No personal-account exposure. - -Risk: - -- Network transfer and runner cost. -- Requires result authentication, archive retention, and reliable toolchain image publication. -- Results are comparable only within the remote environment. - -Effort: 2 to 4 weeks. This is the cleanest long-term answer if the benchmark will run regularly. - -### C. Prefetch and offline cache snapshots - -After generation: - -1. Parse manifests and lockfiles with trusted code. -2. Fetch dependencies into a new cohort cache. -3. Seal the cache snapshot. -4. Run validation offline with the snapshot read-only and a disposable overlay. - -Expected saving: - -- Almost no registry traffic during compile and test phases. -- Registry failures become prefetch infrastructure failures instead of contaminating model verdicts. -- Identical dependency versions are fetched once per cohort. - -Risk: - -- Projects without lockfiles may resolve different versions across dates. -- Some install phases need to execute build backends or download secondary binaries. -- A missing offline artifact must be infrastructure-inconclusive, not a model failure. -- A mirror without an immutable snapshot can reduce comparability rather than improve it. - -Effort: 2 to 4 weeks across all ecosystems. - -### D. Native supervisor - -A Rust supervisor could create process groups, set Darwin policies, stream bounded logs, sample aggregate RSS, enforce PID and output limits, and kill escaped descendants. - -Expected saving: - -- Stronger runaway control and lower harness overhead than repeated shell wrappers. -- Better peak-RSS and CPU accounting. - -Risk: - -- macOS has no cgroup-style aggregate CPU quota. A host supervisor can lower priority and kill over-limit trees, but it cannot match container isolation. -- `taskpolicy -m` and per-process memory limits are not reliable substitutes for an aggregate group limit without careful testing. - -Effort: 1 to 2 weeks. Build it only if the TypeScript process-group executor proves unreliable. - -## Fidelity fixes worth pairing with this work - -| Finding | Change and location | Resource effect | Verdict risk | Effort | -| --------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------- | -------------------------------------- | -----------------------------: | -| Install-only Bun root can pass | In [validateProject()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:348), add an `unvalidated:bun` Core failure when a root has no build, no typecheck, and no covered member that provides one | Negligible | Correctly changes false passes | 0.5 day | -| Java and Elixir select one build root | Replace [findBuildRoot()](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench/validation/index.ts:751) with membership-aware multi-root planning | More work on multi-root output; fail-fast offsets it | Improves coverage but can lower scores | 2 to 3 days | -| Cargo Full gates lack consistent workspace coverage | Add `--workspace --all-targets` to clippy and appropriate workspace coverage to test | More work, not less | Improves Full honesty | 0.5 day | -| Rust Core uses `cargo check`, not linking | Keep `cargo check` for the low-footprint compile gate unless the protocol explicitly requires final linking. If linking becomes required, use `cargo build --workspace --all-targets` and accept the cost | A build is materially heavier | High comparability impact | Protocol decision | -| Python installs all optional extras | Decide whether Core means default plus dev dependencies or every optional extra. If optional extras are not applicable, replace `--all-extras` with locked default sync | Can be a large saving for projects with heavy optional dependencies | Medium to high because verdicts change | 1 day plus documentation | -| Cache archive is lossy | Preserve all authored source and validate a copy instead of deleting every `bin`, `build`, or `dist` directory | Small archive growth | Improves fidelity | Covered by immutable-copy work | - -## Protocol changes required - -Before publishing any result under the optimized validator: - -- Bump `VALIDATION_CACHE_VERSION` from v6. -- Bump the harness patch version. -- Update the protocol table and validator notes in [scaffbench-benchmark.md](/Users/ibrahime/Documents/Better-Fullstack/docs/guidelines/scaffbench-benchmark.md:1). -- Update the protocol guard in [build-scaffbench-3-data.ts](/Users/ibrahime/Documents/Better-Fullstack/scripts/benchmarks/build-scaffbench-3-data.ts:1). -- Add `resourceProfileId`, command-plan digest, sandbox identity, cache mode, and limits to persisted provenance. -- Keep Core and Full independent. A failed advisory command must still permit a Core verdict. A skip must still disqualify Full. -- Increase deadlines with reduced parallelism. -- Re-run canonical recording and weak-versus-strong calibration under the final resource profile. Old validations cannot be mixed with new-profile validations. - -## Recommended adoption order - -1. Immediately use `--generate-only` and a separate `--validate-existing` phase under `taskpolicy`, preferably overnight. -2. Add step telemetry, including peak RSS, CPU time, child count, output bytes, and cooldown time. Pilot one Bun project, the Rust project, and the polyglot project. -3. Fix process-group termination and bounded output capture. -4. Add the fixed two-worker `low-v7` profile, background QoS, longer timeouts, and full provenance. -5. Add planned fail-fast execution with cheap quality gates first. -6. Validate an APFS copy-on-write clone, preserve the archive, use `bun install --no-save`, and replace copied Go tidy with `go mod tidy -diff`. -7. Add `uv run --no-sync`, toolchain-probe memoization, complete cache identity, and a shared result cache. -8. Add the single manifest inventory and artifact budgets. -9. Pilot superset-command reuse for Cargo, .NET, Maven, and Gradle. Adopt each only after equivalence tests. -10. Before the first public 3.0 run, move untrusted validation to a dedicated account at minimum. Prefer a pinned container or remote worker if ScaffBench will become a recurring benchmark. - -I would not start with local mirrors, shared compiler-output directories, or a custom Rust supervisor. Worker caps, fail-fast execution, immutable validation copies, and correct process cleanup offer more benefit for less code and less verdict risk. - -No files were modified, and no tests or builds were run. diff --git a/testing/scaffbench-validator-audit-2026-07-10.md b/testing/scaffbench-validator-audit-2026-07-10.md deleted file mode 100644 index 6c133101d..000000000 --- a/testing/scaffbench-validator-audit-2026-07-10.md +++ /dev/null @@ -1,169 +0,0 @@ -# ScaffBench validation-harness audit - -I reviewed the current working tree and excluded the two supplied bugs. The namespaced `GATE` matching is correct; I found no suffix-classification defect there. - -## Critical - -### 1. Nested manifests are assumed to belong to the shallowest workspace without proving membership - -**Location:** [scaffbench-v2-lib.ts:2931](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2931), [scaffbench-v2-lib.ts:2971](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2971), [scaffbench-v2-lib.ts:2993](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2993) - -**Scenario:** `dropNestedRoots` removes every nested manifest whenever a shallower manifest exists. It does not inspect Bun `workspaces`, Cargo workspace members, or uv workspace configuration. The Bun fallback descends only when the root has neither build nor typecheck; a root `"build": "echo ok"` suppresses validation of independent nested applications. Cargo and Python have no fallback at all. - -**Direction:** False pass. - -**Fix:** Parse actual workspace membership. Validate every nested manifest that is not demonstrably covered by the parent workspace’s build/check command. - -### 2. .NET validates one selected `.csproj`, not the solution - -**Location:** [scaffbench-v2-lib.ts:3445](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3445), [scaffbench-v2-lib.ts:3537](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3537) - -**Scenario:** All `.csproj` directories are discovered, but only `apps/server` or `roots[0]` is restored and built. A solution containing an API, worker, shared library, and test project passes if the selected API builds, even when an unreferenced worker or another solution project does not compile. `.sln`/`.slnx` files are never considered. - -**Direction:** False pass. - -**Fix:** Prefer building the solution file. Without a solution, validate every independent project root and namespace its steps. - -### 3. Python “typecheck” is only a syntax compilation - -**Location:** [scaffbench-v2-lib.ts:3367](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3367) - -**Scenario:** The core Python check is `python -m compileall`. Bytecode compilation does not resolve imports or execute module initialization. A FastAPI app containing `from undeclared_package import Client`, an invalid Pydantic model declaration, or an import-time configuration error can pass install and compileall while failing immediately when imported or started. - -**Direction:** False pass. - -**Fix:** Run a configured typechecker when present and add an import/startup smoke test for the application entry point. Keep `compileall` as an additional syntax check, not the core typecheck. - -### 4. Every validation timeout is classified as infrastructure - -**Location:** [scaffbench-v2-lib.ts:4138](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:4138) - -**Scenario:** Any timed-out step makes the entire run infra-inconclusive. A model-authored `"build": "vite build --watch"`, deadlocked application test, or hanging code generator is therefore removed from the denominator rather than scored as a failure. Conversely, a Docker-dependent advisory test timing out also erases a successfully measured core build. - -**Direction:** Infra-misattribution in both directions. - -**Fix:** Treat core command timeouts as model failures unless there is positive evidence of machine-level failure. Track quality-tier inconclusiveness separately so an advisory timeout does not invalidate an already measured core verdict. - -### 5. Fast network and registry failures are model failures-and then get cached - -**Location:** [scaffbench-v2-lib.ts:3108](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3108), [scaffbench-v2-lib.ts:4138](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:4138) - -**Scenario:** DNS failures, proxy/TLS errors, HTTP 429/5xx responses, registry outages, or corrupted shared Maven/Gradle caches commonly exit nonzero without timing out or failing to spawn. They are classified as model failures. `cacheableValidation` excludes only timeouts and spawn errors, so that transient failure is persisted and replayed for identical sources. - -**Direction:** Infra-misattribution and persistent false fail. - -**Fix:** Recognize narrowly defined transient infrastructure signatures and do not cache those results. Prefer caching deterministic green validations; cache deterministic failures only after classifying their cause. - -### 6. Validation runs on a lossy archive, not the generated project - -**Location:** [scaffbench-v2-lib.ts:2052](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2052), [scaffbench-v2-lib.ts:2897](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2897) - -**Scenario:** Before validation, every directory named `build`, `dist`, `bin`, or `obj` is removed at any depth. These names are not guaranteed to be artifacts. For example, a Go project whose commands live under `bin/server` can be archived with all Go source removed; `go build ./...` may then succeed with no matched packages. Conversely, a package intentionally consuming committed `dist` output can fail only because the harness deleted it. - -**Direction:** False pass or false fail. - -**Fix:** Exclude ecosystem-specific artifact paths only when they are known build outputs. Prefer validating the original isolated tree, then archive it afterward. - -## High impact - -### 7. The three-root cap silently ignores valid services - -**Location:** [scaffbench-v2-lib.ts:2945](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2945), [scaffbench-v2-lib.ts:2972](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2972), [scaffbench-v2-lib.ts:3018](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3018) - -**Scenario:** Only the first three roots per Bun/Cargo/Python/Go ecosystem are validated. A four-service Go monorepo passes even if the lexically later fourth module does not compile. No skipped-root step or inconclusive marker is emitted. - -**Direction:** False pass. - -**Fix:** Remove the cap or batch all roots under the global validation budget. If a cap is necessary, emit a failing/inconclusive core step listing every unvalidated root. - -### 8. `cargo check` never proves that the Rust application links - -**Location:** [scaffbench-v2-lib.ts:3344](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3344) - -**Scenario:** `cargo check` type-checks and compiles metadata but does not link the final binaries. Missing native libraries, undefined extern symbols, and final-link configuration errors can pass `cargo check` while `cargo build` fails. It also lacks explicit `--workspace --all-targets`, leaving coverage to Cargo’s default-member behavior. - -**Direction:** False pass. - -**Fix:** Use `cargo build --workspace --all-targets` as the core build gate, optionally retaining `cargo check` as a faster diagnostic step. - -### 9. React Native/Expo is never bundled or checked by Expo - -**Location:** [scaffbench-v2-lib.ts:3173](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3173), [scaffbench-v2-lib.ts:3243](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3243) - -**Scenario:** Expo projects receive Bun install plus a TypeScript check when available. The “doctor” command is Better-Fullstack’s doctor, not `expo-doctor`. An app with invalid Expo config, incompatible SDK/native-module versions, unresolved Metro assets, or a bundling-only error can pass core validation. - -**Direction:** False pass. - -**Fix:** Add an Expo profile that runs `expo-doctor` and a non-interactive production export/bundle command such as `expo export`. - -### 10. `go mod tidy` promotes test-only problems into the core install gate - -**Location:** [scaffbench-v2-lib.ts:3407](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3407) - -**Scenario:** `go mod tidy` loads the module’s package graph, including test and platform-tagged dependencies, and mutates `go.mod`/`go.sum`. A production application that builds successfully can fail core validation because a `_test.go` file references an unavailable module, even though tests are advisory. It can also repair dependency metadata before the actual build is measured. - -**Direction:** False fail, with possible laundering of manifest defects. - -**Fix:** Use a non-mutating dependency/download step before `go build ./...`. Run tidy as a separate read-only diff/quality check on a copy. - -### 11. Nested missing toolchains are charged to the model, while broken local wrappers are excluded as infra - -**Location:** [scaffbench-v2-lib.ts:2583](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2583), [scaffbench-v2-lib.ts:3493](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3493), [scaffbench-v2-lib.ts:4152](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:4152) - -**Scenario:** If `bun run build` invokes an unavailable system `protoc`, Bun spawns successfully, so `spawnError` is false and exit 127 becomes a model failure. In the opposite direction, an existing model-created `mvnw` without its executable bit produces an `EACCES` spawn error and becomes infra-inconclusive even though the broken mode is part of the artifact. - -**Direction:** Infra-misattribution in both directions. - -**Fix:** Record spawn error codes and command ownership. Missing harness/system executables may be infra; missing executable bits or malformed project-local wrappers should be model failures. Preflight required external codegen tools explicitly. - -### 12. Project disambiguation omits Java, .NET, and Elixir manifests - -**Location:** [scaffbench-v2-lib.ts:2871](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:2871) - -**Scenario:** When the expected project-name directory is absent and multiple candidate directories exist, disambiguation recognizes package.json, Cargo, Go, Python, and `bts.jsonc` only. A real Java `pom.xml`, Gradle, `.csproj`, or `mix.exs` project plus one stray directory resolves to `null` and is scored project-not-found. - -**Direction:** False fail. - -**Fix:** Make candidate detection ecosystem-aware and include `pom.xml`, Gradle manifests, `mix.exs`, solution/project suffixes, and other profiles supported by the validator. - -### 13. Required cross-language code generation is not itself a gate - -**Location:** [scaffbench-v2-lib.ts:3173](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3173), [scaffbench-v2-lib.ts:3407](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3407) - -**Scenario:** The frontier protobuf spec explicitly requires working shared codegen. Bun runs only `build`, Go runs `go build`, and no generic root `generate`, Makefile, Buf, or Taskfile lifecycle is exercised. Checked-in stale/manual stubs can let all component builds pass while `make generate` or `buf generate` is broken. - -**Direction:** False pass. - -**Fix:** Add spec/profile-declared prerequisite commands and run codegen from a clean generated-output state before component builds. - -### 14. Cache identity ignores executable modes, symlinks, platform, and toolchain versions - -**Location:** [scaffbench-v2-lib.ts:3112](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3112), [scaffbench-v2-lib.ts:3128](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:3128) - -**Scenario:** The source hash contains paths and bytes only. It omits executable bits and skips symlinks because only `Dirent.isFile()` entries are collected. Thus executable and non-executable `mvnw` trees collide. The cache key also lacks OS/architecture and Bun, Go, Rust, Python, .NET, Java, and Elixir toolchain versions, so a result can survive an environment change that changes the real verdict. - -**Direction:** False pass or false fail through stale/colliding cache hits. - -**Fix:** Hash file type, mode, and symlink target. Add a validator-plan digest plus platform and relevant toolchain fingerprints to the cache key. - -## Scoring-consumer correctness - -### 15. “Full pass” is true when no quality gate ran - -**Location:** [scaffbench-v2-lib.ts:4115](/Users/ibrahime/Documents/Better-Fullstack/scripts/scaffbench-v2-lib.ts:4115), [build-scaffbench-data.ts:85](/Users/ibrahime/Documents/Better-Fullstack/scripts/benchmarks/build-scaffbench-data.ts:85) - -**Scenario:** `qualityPassed` applies `every()` to an empty advisory-step list. The consumer’s `fullPass` similarly requires only that all existing applicable steps pass; core steps make the list nonempty. With `qualityGate: false`, a native project can therefore receive Full pass without lint, format, or tests having run. TypeScript is inconsistently stricter because a shipped lint script runs even when the quality option is disabled. - -**Direction:** False pass in the Full tier and cross-ecosystem skew. - -**Fix:** Persist whether quality validation was requested and whether expected gates were attempted. Report Full as unavailable when the gate was disabled, or require explicit advisory coverage before Full can be true. - -### 16. Multiple efforts or trials collapse to one arbitrary result in leaderboard consumers - -**Location:** [build-scaffbench-data.ts:113](/Users/ibrahime/Documents/Better-Fullstack/scripts/benchmarks/build-scaffbench-data.ts:113), [splice-scaffbench-2-1.ts:86](/Users/ibrahime/Documents/Better-Fullstack/scripts/benchmarks/splice-scaffbench-2-1.ts:86) - -**Scenario:** `resByCell` is keyed only by `path|specId`, omitting effort and trial. Later results overwrite earlier ones. Meanwhile `scoredRuns`, wired score, cost, and other fields come from the aggregate. A multi-effort or repeated run can therefore publish the last result’s core/full verdict alongside another effort’s identity and an aggregate from several trials; array order can flip the cell. - -**Direction:** False pass or false fail in published leaderboard data. - -**Fix:** Key results by model, effort, path, spec, and trial. For repeated runs, derive the displayed verdict from the aggregate’s pass counts or explicitly assert `repeats === 1` for Pass@1 exports.