feat: show test request and response payloads

dashboard/src/app.css

@@ -2470,6 +2470,8 @@ button.small,
     margin-bottom: 16px;
     display: flex;
     justify-content: flex-end;
+    gap: 8px;
+    flex-wrap: wrap;
 }
 
 .detail-label {
@@ -2482,6 +2484,41 @@ button.small,
     margin-top: 0;
 }
 
+.no-failing-tests {
+    margin: 0 0 16px;
+    text-align: center;
+}
+
+.payload-header {
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: 8px;
+}
+
+.payload-header .detail-label {
+    margin-bottom: 4px;
+}
+
+.btn-copy-payload {
+    padding: 3px 8px;
+    font-size: var(--text-sm);
+    background: var(--color-bg);
+    border: 1px solid var(--color-border);
+    border-radius: var(--radius-sm);
+    color: var(--color-text);
+    cursor: pointer;
+}
+
+.btn-copy-payload:hover:not(:disabled) {
+    background: var(--color-border);
+}
+
+.btn-copy-payload:disabled {
+    cursor: not-allowed;
+    opacity: 0.55;
+}
+
 .runs-badges {
     display: flex;
     flex-wrap: wrap;

dashboard/src/routes/+page.svelte

@@ -28,6 +28,7 @@
     let detailsModalOpen = $state(false);
     let detailsLlm = $state<LLMResult | null>(null);
     let showAllRuns = $state(false);
+    let onlyShowFailingTests = $state(false);
     let selectedRunId = $state<string | null>(null);
     let displayedRunId = $state<string | null>(null);
     let runHistoryBySuite = $state<Record<string, TestRunEntry[]>>({});
@@ -200,6 +201,7 @@
         if (llm) {
             detailsLlm = llm;
             showAllRuns = false;
+            onlyShowFailingTests = false;
             detailsModalOpen = true;
         }
     }
@@ -245,6 +247,26 @@
         }
     }
 
+    function formatPayload(payload: string | undefined): string {
+        if (payload === undefined) return "Payload was not captured for this provider.";
+        return formatJSON(payload);
+    }
+
+    function hasFailingRun(testCase: LLMResult["testCaseResults"][number]): boolean {
+        return testCase.runs.some((run) => !run.isCorrect);
+    }
+
+    async function copyPayload(payload: string | undefined): Promise<void> {
+        if (payload === undefined) return;
+
+        try {
+            await navigator.clipboard.writeText(payload);
+            showSuccess("Payload copied to clipboard");
+        } catch {
+            showError("Could not copy payload to clipboard");
+        }
+    }
+
     function getEvaluationModelValue(model: SelectedModel | undefined): string {
         if (!model) return "";
         return `${model.provider}:${model.modelId}`;
@@ -596,8 +618,21 @@
                 >
                     {showAllRuns ? "Hide individual runs" : "Show individual runs"}
                 </button>
+                <button
+                    type="button"
+                    class="btn-toggle-runs"
+                    class:active={onlyShowFailingTests}
+                    aria-pressed={onlyShowFailingTests}
+                    onclick={() => (onlyShowFailingTests = !onlyShowFailingTests)}
+                >
+                    Only show failing tests
+                </button>
             </div>
-            {#each detailsLlm.testCaseResults as tc, i}
+            {#if onlyShowFailingTests && !detailsLlm.testCaseResults.some(hasFailingRun)}
+                <p class="muted no-failing-tests">All test cases passed.</p>
+            {/if}
+            {#each onlyShowFailingTests ? detailsLlm.testCaseResults.filter(hasFailingRun) : detailsLlm.testCaseResults as tc, i}
+                {@const displayedRun = tc.runs[0]}
                 <div class="test-case-detail">
                     <div class="header">
                         <strong>#{i + 1}</strong>
@@ -611,6 +646,41 @@
                         <div class="detail-label">Expected:</div>
                         <div class="json-preview">{formatJSON(tc.expectedOutput)}</div>
                     {/if}
+                    {#if !showAllRuns && tc.runs.length === 1 && displayedRun}
+                        <div class="detail-label">Actual results:</div>
+                        <div class="json-preview run-output">
+                            {displayedRun.actualOutput ??
+                                (displayedRun.error ? `Error: ${displayedRun.error}` : "N/A")}
+                        </div>
+                        <div class="payload-header">
+                            <div class="detail-label">Request payload:</div>
+                            <button
+                                type="button"
+                                class="btn-copy-payload"
+                                aria-label="Copy request payload"
+                                disabled={displayedRun.requestPayload === undefined}
+                                onclick={() => copyPayload(displayedRun.requestPayload)}
+                                >Copy</button
+                            >
+                        </div>
+                        <pre class="json-preview run-output">{formatPayload(
+                                displayedRun.requestPayload
+                            )}</pre>
+                        <div class="payload-header">
+                            <div class="detail-label">Response payload:</div>
+                            <button
+                                type="button"
+                                class="btn-copy-payload"
+                                aria-label="Copy response payload"
+                                disabled={displayedRun.responsePayload === undefined}
+                                onclick={() => copyPayload(displayedRun.responsePayload)}
+                                >Copy</button
+                            >
+                        </div>
+                        <pre class="json-preview run-output">{formatPayload(
+                                displayedRun.responsePayload
+                            )}</pre>
+                    {/if}
                     {#if !showAllRuns && tc.runs?.length > 0}
                         <div class="runs-badges">
                             {#each tc.runs as run, runIdx}
@@ -658,9 +728,37 @@
                                     </div>
                                     <div class="detail-label">Actual output:</div>
                                     <div class="json-preview run-output">
-                                        {run.actualOutput ||
+                                        {run.actualOutput ??
                                             (run.error ? `Error: ${run.error}` : "N/A")}
                                     </div>
+                                    <div class="payload-header">
+                                        <div class="detail-label">Request payload:</div>
+                                        <button
+                                            type="button"
+                                            class="btn-copy-payload"
+                                            aria-label="Copy request payload"
+                                            disabled={run.requestPayload === undefined}
+                                            onclick={() => copyPayload(run.requestPayload)}
+                                            >Copy</button
+                                        >
+                                    </div>
+                                    <pre class="json-preview run-output">{formatPayload(
+                                            run.requestPayload
+                                        )}</pre>
+                                    <div class="payload-header">
+                                        <div class="detail-label">Response payload:</div>
+                                        <button
+                                            type="button"
+                                            class="btn-copy-payload"
+                                            aria-label="Copy response payload"
+                                            disabled={run.responsePayload === undefined}
+                                            onclick={() => copyPayload(run.responsePayload)}
+                                            >Copy</button
+                                        >
+                                    </div>
+                                    <pre class="json-preview run-output">{formatPayload(
+                                            run.responsePayload
+                                        )}</pre>
                                     {#if run.reason}
                                         <div class="detail-reason">{run.reason}</div>
                                     {/if}

package.json

@@ -1,6 +1,6 @@
 {
     "name": "relia-prompt",
-    "version": "1.3.0",
+    "version": "1.4.0",
     "description": "Test and benchmark prompts accross LLM providers and models.",
     "repository": {
         "type": "git",

shared/types.ts

@@ -163,6 +163,8 @@ export interface TestRun {
     score: number;
     isCorrect: boolean;
     actualOutput?: string;
+    requestPayload?: string;
+    responsePayload?: string;
     error?: string;
     durationMs?: number;
     expectedFound?: number;

src/errors.ts

@@ -50,10 +50,19 @@ export class ConfigurationError extends AppError {
  */
 export class LLMError extends AppError {
     public readonly provider: string;
+    public readonly requestPayload?: string;
+    public readonly responsePayload?: string;
 
-    constructor(provider: string, message: string) {
+    constructor(
+        provider: string,
+        message: string,
+        requestPayload?: string,
+        responsePayload?: string
+    ) {
         super(`[${provider}] ${message}`, 502);
         this.provider = provider;
+        this.requestPayload = requestPayload;
+        this.responsePayload = responsePayload;
     }
 }
 

src/llm-clients/cerebras-client.ts

@@ -1,4 +1,4 @@
-import { LLMClient, ModelInfo } from "./llm-client";
+import { LLMClient, LLMCompletionOptions, LLMCompletionTrace, ModelInfo } from "./llm-client";
 import { formatModelName } from "./utils";
 import { getConfig } from "../runtime/config";
 import { ConfigurationError, LLMError } from "../errors";
@@ -14,6 +14,38 @@ interface CerebrasModelsResponse {
     data: CerebrasModel[];
 }
 
+const unsupportedStrictSchemaKeywords = new Set([
+    "$schema",
+    "default",
+    "description",
+    "examples",
+    "format",
+    "maxItems",
+    "minItems",
+    "pattern",
+    "title",
+]);
+
+export function toCerebrasStrictSchema(schema: Record<string, unknown>): Record<string, unknown> {
+    return removeUnsupportedStrictSchemaKeywords(schema) as Record<string, unknown>;
+}
+
+function removeUnsupportedStrictSchemaKeywords(value: unknown): unknown {
+    if (Array.isArray(value)) {
+        return value.map(removeUnsupportedStrictSchemaKeywords);
+    }
+
+    if (!value || typeof value !== "object") {
+        return value;
+    }
+
+    return Object.fromEntries(
+        Object.entries(value)
+            .filter(([key]) => !unsupportedStrictSchemaKeywords.has(key))
+            .map(([key, nestedValue]) => [key, removeUnsupportedStrictSchemaKeywords(nestedValue)])
+    );
+}
+
 export class CerebrasClient implements LLMClient {
     providerId = "cerebras";
     private baseUrl = "https://api.cerebras.ai/v1";
@@ -59,8 +91,9 @@ export class CerebrasClient implements LLMClient {
         messages: Array<{ role: "system" | "user"; content: string }>,
         temperature: number,
         modelId: string,
-        defaultValue: string = ""
-    ): Promise<string> {
+        defaultValue: string = "",
+        options?: LLMCompletionOptions
+    ): Promise<LLMCompletionTrace> {
         const apiKey = this.getApiKey();
         if (!apiKey) {
             throw new ConfigurationError("Cerebras API key not configured");
@@ -71,38 +104,68 @@ export class CerebrasClient implements LLMClient {
             messages,
             temperature,
             max_tokens: 4096,
-            response_format: { type: "json_object" },
+            response_format: options?.responseSchema
+                ? {
+                      type: "json_schema",
+                      json_schema: {
+                          name: "nls_response",
+                          strict: true,
+                          schema: toCerebrasStrictSchema(options.responseSchema),
+                      },
+                  }
+                : { type: "json_object" },
         };
+        const requestPayload = JSON.stringify(requestBody);
 
         const response = await fetch(`${this.baseUrl}/chat/completions`, {
             method: "POST",
             headers: {
                 "Content-Type": "application/json",
                 Authorization: `Bearer ${apiKey}`,
             },
-            body: JSON.stringify(requestBody),
+            body: requestPayload,
         });
+        const responsePayload = await response.text();
 
         if (!response.ok) {
-            const error = await response.text();
-            throw new LLMError("Cerebras", `API error: ${response.status} - ${error}`);
+            throw new LLMError(
+                "Cerebras",
+                `API error: ${response.status} - ${responsePayload}`,
+                requestPayload,
+                responsePayload
+            );
         }
 
-        const data = (await response.json()) as {
+        const parsedResponsePayload = JSON.parse(responsePayload) as {
             choices?: Array<{ message?: { content?: string } }>;
         };
-        return data.choices?.[0]?.message?.content ?? defaultValue;
+        return {
+            content: parsedResponsePayload.choices?.[0]?.message?.content ?? defaultValue,
+            requestPayload,
+            responsePayload,
+        };
     }
 
     async complete(systemPrompt: string, userMessage: string, modelId: string): Promise<string> {
+        const completion = await this.completeWithTrace(systemPrompt, userMessage, modelId);
+        return completion.content;
+    }
+
+    async completeWithTrace(
+        systemPrompt: string,
+        userMessage: string,
+        modelId: string,
+        options?: LLMCompletionOptions
+    ): Promise<LLMCompletionTrace> {
         return this.makeRequest(
             [
                 { role: "system", content: systemPrompt },
                 { role: "user", content: userMessage },
             ],
             0.1,
             modelId,
-            ""
+            "",
+            options
         );
     }
 }

src/llm-clients/cerebras-structured-output.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, test } from "bun:test";
+import { toCerebrasStrictSchema } from "./cerebras-client";
+
+describe("toCerebrasStrictSchema", () => {
+    test("removes keywords unsupported by Cerebras strict structured output", () => {
+        expect(
+            toCerebrasStrictSchema({
+                $schema: "https://json-schema.org/draft/2020-12/schema",
+                type: "object",
+                description: "Response",
+                properties: {
+                    dates: {
+                        type: "array",
+                        minItems: 1,
+                        maxItems: 2,
+                        items: { type: "string", format: "date", pattern: "^\\d+$" },
+                    },
+                },
+            })
+        ).toEqual({
+            type: "object",
+            properties: {
+                dates: {
+                    type: "array",
+                    items: { type: "string" },
+                },
+            },
+        });
+    });
+});