From e347174af41ebed54f8a34c720dcfda6b41d4ee6 Mon Sep 17 00:00:00 2001 From: Fabien Ruffin Date: Tue, 21 Jul 2026 16:58:04 +1000 Subject: [PATCH 1/3] feat: show test request and response payloads --- dashboard/src/app.css | 37 +++++++ dashboard/src/routes/+page.svelte | 102 +++++++++++++++++- shared/types.ts | 2 + src/errors.ts | 11 +- src/llm-clients/cerebras-client.ts | 52 +++++++-- .../cerebras-structured-output.test.ts | 30 ++++++ src/llm-clients/cerebras-structured-output.ts | 31 ++++++ src/llm-clients/groq-client.ts | 33 ++++-- src/llm-clients/llm-client.ts | 16 +++ src/services/test-runner.test.ts | 59 +++++++++- src/services/test-runner.ts | 55 +++++++--- 11 files changed, 394 insertions(+), 34 deletions(-) create mode 100644 src/llm-clients/cerebras-structured-output.test.ts create mode 100644 src/llm-clients/cerebras-structured-output.ts diff --git a/dashboard/src/app.css b/dashboard/src/app.css index 6db63b1..f487331 100644 --- a/dashboard/src/app.css +++ b/dashboard/src/app.css @@ -2470,6 +2470,8 @@ button.small, margin-bottom: 16px; display: flex; justify-content: flex-end; + gap: 8px; + flex-wrap: wrap; } .detail-label { @@ -2482,6 +2484,41 @@ button.small, margin-top: 0; } +.no-failing-tests { + margin: 0 0 16px; + text-align: center; +} + +.payload-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; +} + +.payload-header .detail-label { + margin-bottom: 4px; +} + +.btn-copy-payload { + padding: 3px 8px; + font-size: var(--text-sm); + background: var(--color-bg); + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + color: var(--color-text); + cursor: pointer; +} + +.btn-copy-payload:hover:not(:disabled) { + background: var(--color-border); +} + +.btn-copy-payload:disabled { + cursor: not-allowed; + opacity: 0.55; +} + .runs-badges { display: flex; flex-wrap: wrap; diff --git a/dashboard/src/routes/+page.svelte b/dashboard/src/routes/+page.svelte index 20d1873..fd9fa7c 100644 --- a/dashboard/src/routes/+page.svelte +++ b/dashboard/src/routes/+page.svelte @@ -28,6 +28,7 @@ let detailsModalOpen = $state(false); let detailsLlm = $state(null); let showAllRuns = $state(false); + let onlyShowFailingTests = $state(false); let selectedRunId = $state(null); let displayedRunId = $state(null); let runHistoryBySuite = $state>({}); @@ -200,6 +201,7 @@ if (llm) { detailsLlm = llm; showAllRuns = false; + onlyShowFailingTests = false; detailsModalOpen = true; } } @@ -245,6 +247,26 @@ } } + function formatPayload(payload: string | undefined): string { + if (payload === undefined) return "Payload was not captured for this provider."; + return formatJSON(payload); + } + + function hasFailingRun(testCase: LLMResult["testCaseResults"][number]): boolean { + return testCase.runs.some((run) => !run.isCorrect); + } + + async function copyPayload(payload: string | undefined): Promise { + if (payload === undefined) return; + + try { + await navigator.clipboard.writeText(payload); + showSuccess("Payload copied to clipboard"); + } catch { + showError("Could not copy payload to clipboard"); + } + } + function getEvaluationModelValue(model: SelectedModel | undefined): string { if (!model) return ""; return `${model.provider}:${model.modelId}`; @@ -596,8 +618,21 @@ > {showAllRuns ? "Hide individual runs" : "Show individual runs"} + - {#each detailsLlm.testCaseResults as tc, i} + {#if onlyShowFailingTests && !detailsLlm.testCaseResults.some(hasFailingRun)} +

All test cases passed.

+ {/if} + {#each onlyShowFailingTests ? detailsLlm.testCaseResults.filter(hasFailingRun) : detailsLlm.testCaseResults as tc, i} + {@const displayedRun = tc.runs[0]}
#{i + 1} @@ -611,6 +646,41 @@
Expected:
{formatJSON(tc.expectedOutput)}
{/if} + {#if !showAllRuns && tc.runs.length === 1 && displayedRun} +
Actual results:
+
+ {displayedRun.actualOutput ?? + (displayedRun.error ? `Error: ${displayedRun.error}` : "N/A")} +
+
+
Request payload:
+ +
+ 
{formatPayload(
+                                displayedRun.requestPayload
+                            )}
+
+
Response payload:
+ +
+ 
{formatPayload(
+                                displayedRun.responsePayload
+                            )}
+ {/if} {#if !showAllRuns && tc.runs?.length > 0}
{#each tc.runs as run, runIdx} @@ -658,9 +728,37 @@
Actual output:
- {run.actualOutput || + {run.actualOutput ?? (run.error ? `Error: ${run.error}` : "N/A")}
+
+
Request payload:
+ +
+ 
{formatPayload(
+                                            run.requestPayload
+                                        )}
+
+
Response payload:
+ +
+ 
{formatPayload(
+                                            run.responsePayload
+                                        )}
{#if run.reason}
{run.reason}
{/if} diff --git a/shared/types.ts b/shared/types.ts index 56d6188..e487a36 100644 --- a/shared/types.ts +++ b/shared/types.ts @@ -163,6 +163,8 @@ export interface TestRun { score: number; isCorrect: boolean; actualOutput?: string; + requestPayload?: string; + responsePayload?: string; error?: string; durationMs?: number; expectedFound?: number; diff --git a/src/errors.ts b/src/errors.ts index a767b2a..34a8ee9 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -50,10 +50,19 @@ export class ConfigurationError extends AppError { */ export class LLMError extends AppError { public readonly provider: string; + public readonly requestPayload?: string; + public readonly responsePayload?: string; - constructor(provider: string, message: string) { + constructor( + provider: string, + message: string, + requestPayload?: string, + responsePayload?: string + ) { super(`[${provider}] ${message}`, 502); this.provider = provider; + this.requestPayload = requestPayload; + this.responsePayload = responsePayload; } } diff --git a/src/llm-clients/cerebras-client.ts b/src/llm-clients/cerebras-client.ts index 66536ca..27faecc 100644 --- a/src/llm-clients/cerebras-client.ts +++ b/src/llm-clients/cerebras-client.ts @@ -1,4 +1,5 @@ -import { LLMClient, ModelInfo } from "./llm-client"; +import { LLMClient, LLMCompletionOptions, LLMCompletionTrace, ModelInfo } from "./llm-client"; +import { toCerebrasStrictSchema } from "./cerebras-structured-output"; import { formatModelName } from "./utils"; import { getConfig } from "../runtime/config"; import { ConfigurationError, LLMError } from "../errors"; @@ -59,8 +60,9 @@ export class CerebrasClient implements LLMClient { messages: Array<{ role: "system" | "user"; content: string }>, temperature: number, modelId: string, - defaultValue: string = "" - ): Promise { + defaultValue: string = "", + options?: LLMCompletionOptions + ): Promise { const apiKey = this.getApiKey(); if (!apiKey) { throw new ConfigurationError("Cerebras API key not configured"); @@ -71,8 +73,18 @@ export class CerebrasClient implements LLMClient { messages, temperature, max_tokens: 4096, - response_format: { type: "json_object" }, + response_format: options?.responseSchema + ? { + type: "json_schema", + json_schema: { + name: "nls_response", + strict: true, + schema: toCerebrasStrictSchema(options.responseSchema), + }, + } + : { type: "json_object" }, }; + const requestPayload = JSON.stringify(requestBody); const response = await fetch(`${this.baseUrl}/chat/completions`, { method: "POST", @@ -80,21 +92,40 @@ export class CerebrasClient implements LLMClient { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, - body: JSON.stringify(requestBody), + body: requestPayload, }); + const responsePayload = await response.text(); if (!response.ok) { - const error = await response.text(); - throw new LLMError("Cerebras", `API error: ${response.status} - ${error}`); + throw new LLMError( + "Cerebras", + `API error: ${response.status} - ${responsePayload}`, + requestPayload, + responsePayload + ); } - const data = (await response.json()) as { + const parsedResponsePayload = JSON.parse(responsePayload) as { choices?: Array<{ message?: { content?: string } }>; }; - return data.choices?.[0]?.message?.content ?? defaultValue; + return { + content: parsedResponsePayload.choices?.[0]?.message?.content ?? defaultValue, + requestPayload, + responsePayload, + }; } async complete(systemPrompt: string, userMessage: string, modelId: string): Promise { + const completion = await this.completeWithTrace(systemPrompt, userMessage, modelId); + return completion.content; + } + + async completeWithTrace( + systemPrompt: string, + userMessage: string, + modelId: string, + options?: LLMCompletionOptions + ): Promise { return this.makeRequest( [ { role: "system", content: systemPrompt }, @@ -102,7 +133,8 @@ export class CerebrasClient implements LLMClient { ], 0.1, modelId, - "" + "", + options ); } } diff --git a/src/llm-clients/cerebras-structured-output.test.ts b/src/llm-clients/cerebras-structured-output.test.ts new file mode 100644 index 0000000..66f6058 --- /dev/null +++ b/src/llm-clients/cerebras-structured-output.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, test } from "bun:test"; +import { toCerebrasStrictSchema } from "./cerebras-structured-output"; + +describe("toCerebrasStrictSchema", () => { + test("removes keywords unsupported by Cerebras strict structured output", () => { + expect( + toCerebrasStrictSchema({ + $schema: "https://json-schema.org/draft/2020-12/schema", + type: "object", + description: "Response", + properties: { + dates: { + type: "array", + minItems: 1, + maxItems: 2, + items: { type: "string", format: "date", pattern: "^\\d+$" }, + }, + }, + }) + ).toEqual({ + type: "object", + properties: { + dates: { + type: "array", + items: { type: "string" }, + }, + }, + }); + }); +}); diff --git a/src/llm-clients/cerebras-structured-output.ts b/src/llm-clients/cerebras-structured-output.ts new file mode 100644 index 0000000..9bc99ab --- /dev/null +++ b/src/llm-clients/cerebras-structured-output.ts @@ -0,0 +1,31 @@ +const unsupportedStrictSchemaKeywords = new Set([ + "$schema", + "default", + "description", + "examples", + "format", + "maxItems", + "minItems", + "pattern", + "title", +]); + +export function toCerebrasStrictSchema(schema: Record): Record { + return removeUnsupportedStrictSchemaKeywords(schema) as Record; +} + +function removeUnsupportedStrictSchemaKeywords(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(removeUnsupportedStrictSchemaKeywords); + } + + if (!value || typeof value !== "object") { + return value; + } + + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => !unsupportedStrictSchemaKeywords.has(key)) + .map(([key, nestedValue]) => [key, removeUnsupportedStrictSchemaKeywords(nestedValue)]) + ); +} diff --git a/src/llm-clients/groq-client.ts b/src/llm-clients/groq-client.ts index ef8cb13..c3ba543 100644 --- a/src/llm-clients/groq-client.ts +++ b/src/llm-clients/groq-client.ts @@ -1,4 +1,4 @@ -import { LLMClient, ModelInfo } from "./llm-client"; +import { LLMClient, LLMCompletionTrace, ModelInfo } from "./llm-client"; import { formatModelName } from "./utils"; import { getConfig } from "../runtime/config"; import { ConfigurationError, LLMError } from "../errors"; @@ -62,7 +62,7 @@ export class GroqClient implements LLMClient { temperature: number, modelId: string, defaultValue: string = "" - ): Promise { + ): Promise { const apiKey = this.getApiKey(); if (!apiKey) { throw new ConfigurationError("Groq API key not configured"); @@ -75,6 +75,7 @@ export class GroqClient implements LLMClient { max_tokens: 4096, response_format: { type: "json_object" }, }; + const requestPayload = JSON.stringify(requestBody); const response = await fetch(`${this.baseUrl}/chat/completions`, { method: "POST", @@ -82,21 +83,39 @@ export class GroqClient implements LLMClient { "Content-Type": "application/json", Authorization: `Bearer ${apiKey}`, }, - body: JSON.stringify(requestBody), + body: requestPayload, }); + const responsePayload = await response.text(); if (!response.ok) { - const error = await response.text(); - throw new LLMError("Groq", `API error: ${response.status} - ${error}`); + throw new LLMError( + "Groq", + `API error: ${response.status} - ${responsePayload}`, + requestPayload, + responsePayload + ); } - const data = (await response.json()) as { + const parsedResponsePayload = JSON.parse(responsePayload) as { choices?: Array<{ message?: { content?: string } }>; }; - return data.choices?.[0]?.message?.content ?? defaultValue; + return { + content: parsedResponsePayload.choices?.[0]?.message?.content ?? defaultValue, + requestPayload, + responsePayload, + }; } async complete(systemPrompt: string, userMessage: string, modelId: string): Promise { + const completion = await this.completeWithTrace(systemPrompt, userMessage, modelId); + return completion.content; + } + + async completeWithTrace( + systemPrompt: string, + userMessage: string, + modelId: string + ): Promise { return this.makeRequest( [ { role: "system", content: systemPrompt }, diff --git a/src/llm-clients/llm-client.ts b/src/llm-clients/llm-client.ts index e4a2a49..b592485 100644 --- a/src/llm-clients/llm-client.ts +++ b/src/llm-clients/llm-client.ts @@ -6,11 +6,27 @@ export interface ModelInfo { export type ProviderId = string; +export interface LLMCompletionTrace { + content: string; + requestPayload?: string; + responsePayload?: string; +} + +export interface LLMCompletionOptions { + responseSchema?: Record; +} + export interface LLMClient { providerId: ProviderId; isConfigured(): boolean; listModels(): Promise; complete(systemPrompt: string, userMessage: string, modelId: string): Promise; + completeWithTrace?( + systemPrompt: string, + userMessage: string, + modelId: string, + options?: LLMCompletionOptions + ): Promise; refresh(): void; } diff --git a/src/services/test-runner.test.ts b/src/services/test-runner.test.ts index b123405..8f47528 100644 --- a/src/services/test-runner.test.ts +++ b/src/services/test-runner.test.ts @@ -159,10 +159,67 @@ describe("test-runner", () => { expect(mockClient.complete).toHaveBeenCalled(); const [systemPrompt] = (mockClient.complete as ReturnType).mock .calls[0] as [string, string, string]; - expect(systemPrompt).toContain("## Response Schema:"); + expect(systemPrompt).toContain("## Output schema:"); expect(systemPrompt).toContain('"type":"object"'); }); + test("passes the schema to Cerebras without adding it to the prompt", async () => { + const mockClient = createMockLLMClient("cerebras"); + mockClient.completeWithTrace = mock(() => Promise.resolve({ content: "hello" })); + + const schema = { type: "object", properties: { name: { type: "string" } } }; + const prompt: MinimalPrompt = { + ...createPrompt(1, "Test prompt"), + expectedSchema: JSON.stringify(schema), + }; + const testCases = [createTestCase(1, "input1", "hello", ParseType.STRING)]; + const modelRunners: ModelRunner[] = [ + { + client: mockClient, + modelId: "gpt-oss-120b", + displayName: "cerebras (gpt-oss-120b)", + }, + ]; + + await runTests(prompt, testCases, modelRunners, 1); + + const calls = (mockClient.completeWithTrace as ReturnType).mock.calls; + expect(calls[0]).toEqual([ + "Test prompt", + "input1", + "gpt-oss-120b", + { responseSchema: schema }, + ]); + }); + + test("does not append a schema already included in the prompt", async () => { + const mockClient = createMockLLMClient("test-client"); + mockClient.complete = mock(() => Promise.resolve("hello")); + + const schema = JSON.stringify({ + type: "object", + properties: { name: { type: "string" } }, + }); + const prompt: MinimalPrompt = { + ...createPrompt(1, `Test prompt\n\n## Output schema:\n${schema}`), + expectedSchema: schema, + }; + const testCases = [createTestCase(1, "input1", "hello", ParseType.STRING)]; + const modelRunners: ModelRunner[] = [ + { + client: mockClient, + modelId: "test-model", + displayName: "test-client (test-model)", + }, + ]; + + await runTests(prompt, testCases, modelRunners, 1); + + const [systemPrompt] = (mockClient.complete as ReturnType).mock + .calls[0] as [string, string, string]; + expect(systemPrompt).toBe(prompt.content); + }); + test("should run multiple test cases", async () => { const mockClient = createMockLLMClient("test-client"); let callCount = 0; diff --git a/src/services/test-runner.ts b/src/services/test-runner.ts index bf908cc..9da2ffa 100644 --- a/src/services/test-runner.ts +++ b/src/services/test-runner.ts @@ -1,8 +1,8 @@ import type { EvaluationMode } from "../../shared/types"; -import type { LLMClient, ModelSelection } from "../llm-clients"; +import type { LLMClient, LLMCompletionOptions, ModelSelection } from "../llm-clients"; import { compare } from "../utils/compare"; import { parse, ParseType } from "../utils/parse"; -import { ConfigurationError, getErrorMessage } from "../errors"; +import { ConfigurationError, getErrorMessage, LLMError } from "../errors"; /** Minimal prompt shape for runTests (no DB-only fields). */ export interface MinimalPrompt { @@ -45,6 +45,10 @@ function validate(schemaCandidate: unknown): JsonSchemaObject { return schemaCandidate; } +function promptAlreadyIncludesSchema(promptContent: string, schema: JsonSchemaObject): boolean { + return promptContent.includes(JSON.stringify(schema)); +} + function resolveComparisonParseType( declaredExpectedOutputType: string, expectedOutput: string @@ -130,6 +134,8 @@ export interface EvaluationIssue { */ export interface BaseTestResult { actualOutput: string | null; + requestPayload?: string; + responsePayload?: string; isCorrect: boolean; score: number; // 0-1 score expectedFound: number; @@ -334,8 +340,7 @@ export async function runTests( const schemaString = expectedSchema ?? (typeof prompt === "object" ? prompt.expectedSchema : undefined); - // Build the system prompt with schema hint if present - let systemPrompt = promptContent; + let normalizedSchema: JsonSchemaObject | undefined; if (schemaString) { try { const parsedSchema = JSON.parse(schemaString); @@ -346,10 +351,7 @@ export async function runTests( parsedSchema.schema && typeof parsedSchema.schema === "object" ? parsedSchema.schema : parsedSchema; - const normalizedSchema = validate(schema); - - // Append schema hint to the system prompt - systemPrompt = `${promptContent}\n\n## Response Schema:\n${JSON.stringify(normalizedSchema)}`; + normalizedSchema = validate(schema); } catch { // If parsing fails, ignore the schema console.warn("Failed to parse expectedSchema, ignoring structured output"); @@ -359,6 +361,17 @@ export async function runTests( const llmResults: LLMTestResult[] = []; const llmPromises = modelRunners.map(async (runner) => { + const usesCerebrasStructuredOutput = runner.client.providerId === "cerebras"; + const completionOptions: LLMCompletionOptions | undefined = + usesCerebrasStructuredOutput && normalizedSchema + ? { responseSchema: normalizedSchema } + : undefined; + const systemPrompt = + usesCerebrasStructuredOutput || + !normalizedSchema || + promptAlreadyIncludesSchema(promptContent, normalizedSchema) + ? promptContent + : `${promptContent}\n\n## Output schema:\n${JSON.stringify(normalizedSchema)}`; const testCaseResults: TestCaseResult[] = []; let llmCorrectCount = 0; let llmTotalRuns = 0; @@ -372,11 +385,21 @@ export async function runTests( try { const startTime = Date.now(); // System prompt includes schema hint if present - const actualOutput = await runner.client.complete( - systemPrompt, - testCase.input, - runner.modelId - ); + const completion = runner.client.completeWithTrace + ? await runner.client.completeWithTrace( + systemPrompt, + testCase.input, + runner.modelId, + completionOptions + ) + : { + content: await runner.client.complete( + systemPrompt, + testCase.input, + runner.modelId + ), + }; + const actualOutput = completion.content; const durationMs = Date.now() - startTime; let score = 0; @@ -448,6 +471,8 @@ export async function runTests( runs.push({ runNumber, actualOutput, + requestPayload: completion.requestPayload, + responsePayload: completion.responsePayload, isCorrect, score, expectedFound, @@ -465,6 +490,10 @@ export async function runTests( runs.push({ runNumber, actualOutput: null, + requestPayload: + error instanceof LLMError ? error.requestPayload : undefined, + responsePayload: + error instanceof LLMError ? error.responsePayload : undefined, isCorrect: false, score: 0, expectedFound: 0, From e8fa4dc0a9f4d16a23a74b20b746958b6415118c Mon Sep 17 00:00:00 2001 From: Fabien Ruffin Date: Tue, 21 Jul 2026 17:01:11 +1000 Subject: [PATCH 2/3] chore: bump version to 1.4.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 53b2018..94f505a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "relia-prompt", - "version": "1.3.0", + "version": "1.4.0", "description": "Test and benchmark prompts accross LLM providers and models.", "repository": { "type": "git", From f7afd959937fe668f15e2e322b60e70099d2feef Mon Sep 17 00:00:00 2001 From: Fabien Ruffin Date: Fri, 24 Jul 2026 11:35:58 +1000 Subject: [PATCH 3/3] Fixed PR feedback --- src/llm-clients/cerebras-client.ts | 33 ++++++++++++++++++- .../cerebras-structured-output.test.ts | 2 +- src/llm-clients/cerebras-structured-output.ts | 31 ----------------- 3 files changed, 33 insertions(+), 33 deletions(-) delete mode 100644 src/llm-clients/cerebras-structured-output.ts diff --git a/src/llm-clients/cerebras-client.ts b/src/llm-clients/cerebras-client.ts index 27faecc..7ccb921 100644 --- a/src/llm-clients/cerebras-client.ts +++ b/src/llm-clients/cerebras-client.ts @@ -1,5 +1,4 @@ import { LLMClient, LLMCompletionOptions, LLMCompletionTrace, ModelInfo } from "./llm-client"; -import { toCerebrasStrictSchema } from "./cerebras-structured-output"; import { formatModelName } from "./utils"; import { getConfig } from "../runtime/config"; import { ConfigurationError, LLMError } from "../errors"; @@ -15,6 +14,38 @@ interface CerebrasModelsResponse { data: CerebrasModel[]; } +const unsupportedStrictSchemaKeywords = new Set([ + "$schema", + "default", + "description", + "examples", + "format", + "maxItems", + "minItems", + "pattern", + "title", +]); + +export function toCerebrasStrictSchema(schema: Record): Record { + return removeUnsupportedStrictSchemaKeywords(schema) as Record; +} + +function removeUnsupportedStrictSchemaKeywords(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(removeUnsupportedStrictSchemaKeywords); + } + + if (!value || typeof value !== "object") { + return value; + } + + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => !unsupportedStrictSchemaKeywords.has(key)) + .map(([key, nestedValue]) => [key, removeUnsupportedStrictSchemaKeywords(nestedValue)]) + ); +} + export class CerebrasClient implements LLMClient { providerId = "cerebras"; private baseUrl = "https://api.cerebras.ai/v1"; diff --git a/src/llm-clients/cerebras-structured-output.test.ts b/src/llm-clients/cerebras-structured-output.test.ts index 66f6058..11597a0 100644 --- a/src/llm-clients/cerebras-structured-output.test.ts +++ b/src/llm-clients/cerebras-structured-output.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { toCerebrasStrictSchema } from "./cerebras-structured-output"; +import { toCerebrasStrictSchema } from "./cerebras-client"; describe("toCerebrasStrictSchema", () => { test("removes keywords unsupported by Cerebras strict structured output", () => { diff --git a/src/llm-clients/cerebras-structured-output.ts b/src/llm-clients/cerebras-structured-output.ts deleted file mode 100644 index 9bc99ab..0000000 --- a/src/llm-clients/cerebras-structured-output.ts +++ /dev/null @@ -1,31 +0,0 @@ -const unsupportedStrictSchemaKeywords = new Set([ - "$schema", - "default", - "description", - "examples", - "format", - "maxItems", - "minItems", - "pattern", - "title", -]); - -export function toCerebrasStrictSchema(schema: Record): Record { - return removeUnsupportedStrictSchemaKeywords(schema) as Record; -} - -function removeUnsupportedStrictSchemaKeywords(value: unknown): unknown { - if (Array.isArray(value)) { - return value.map(removeUnsupportedStrictSchemaKeywords); - } - - if (!value || typeof value !== "object") { - return value; - } - - return Object.fromEntries( - Object.entries(value) - .filter(([key]) => !unsupportedStrictSchemaKeywords.has(key)) - .map(([key, nestedValue]) => [key, removeUnsupportedStrictSchemaKeywords(nestedValue)]) - ); -}