From fffc6386b714cb403a1a1954d599e3dc07d7cbd4 Mon Sep 17 00:00:00 2001
From: JNK234 <jwalapuramnarasimha@gmail.com>
Date: Tue, 8 Sep 2026 19:07:51 -0400
Subject: [PATCH] fix: make the llm:choose prompt describe the enum schema it
 is sent with

The choose prompt told the model to reply with only the bare option text
while the request carried an EnumFormat schema requiring {"choice": ...}.
Providers that validate the reply against the schema after generation
(Groq) rejected the whole response with HTTP 400 json_validate_failed, so
llm:choose failed on every call there.

The prompt now asks for the same {"choice": "<option>"} object the schema
enforces. Providers that ignore the constraint still work through the
existing bare-text fallback.

Adds a deterministic test pinning the prompt's final instruction to the
schema shape, and a __TEST_ECHO_PROMPT_TAIL marker in the test provider.
Verified live against Groq gpt-oss-20b: 25/25 e2e assertions pass.
---
 src/main/LLMExtension.scala              | 18 +++++++++++++-----
 src/test/DeterministicTestProvider.scala |  8 +++++++-
 tests.txt                                |  9 +++++++++
 3 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/src/main/LLMExtension.scala b/src/main/LLMExtension.scala
index 54f2cd5..d158727 100644
--- a/src/main/LLMExtension.scala
+++ b/src/main/LLMExtension.scala
@@ -594,6 +594,10 @@ class LLMExtension extends DefaultClassManager {
   }
   
   object ChooseReporter extends Reporter {
+    /** Final line of the choose prompt; names the shape the enum schema enforces. */
+    val ChoiceInstruction: String =
+      s"""Your choice as {"${EnumFormat.ChoiceKey}": "<option>"}:"""
+
     override def getSyntax: Syntax = Syntax.reporterSyntax(
       right = List(Syntax.StringType, Syntax.ListType),
       ret = Syntax.StringType
@@ -613,14 +617,18 @@ class LLMExtension extends DefaultClassManager {
           throw new ExtensionException("Choice list cannot be empty")
         }
 
+        // The wording must describe the same shape as the EnumFormat schema
+        // sent with the request. A provider that validates the reply against
+        // the schema server-side (Groq does) rejects the whole response with
+        // HTTP 400 when the model follows a prompt asking for bare text.
         val systemPrompt = "You are a decision-making assistant. " +
-          "When given options, respond with EXACTLY one option from the list. " +
-          "Rules: Reply with ONLY the option text. " +
-          "No explanation, no numbering, no punctuation, no quotes, no extra words. " +
-          "Copy the option exactly as written."
+          "When given options, choose EXACTLY one option from the list. " +
+          s"""Reply with a JSON object of the form {"${EnumFormat.ChoiceKey}": "<option>"} """ +
+          "where <option> is copied exactly as written. " +
+          "No explanation, no extra keys, no extra text."
 
         val userPrompt = s"$prompt\n\nOptions:\n${choices.mkString("\n")}\n\n" +
-          "Your choice (one option, no other text):"
+          ChooseReporter.ChoiceInstruction
 
         // Build temp history with system prompt — don't mutate permanent history
         val tempHistory = ArrayBuffer.from(snapshotHistory(agent))
diff --git a/src/test/DeterministicTestProvider.scala b/src/test/DeterministicTestProvider.scala
index cca0ecb..cb118e5 100644
--- a/src/test/DeterministicTestProvider.scala
+++ b/src/test/DeterministicTestProvider.scala
@@ -82,6 +82,12 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid
     // primitive that stopped constraining its request fails a test instead of
     // silently degrading. The enum echo answers with the LAST choice because
     // the unconstrained prompt-only path answers with the first.
+    // __TEST_ECHO_PROMPT_TAIL answers with the prompt's final line so a test can
+    // pin the instruction the model actually sees next to the schema it is sent.
+    if (lastUserMessage.contains("__TEST_ECHO_PROMPT_TAIL")) {
+      return Future.successful(structured(lastUserMessage.linesIterator.toSeq.last))
+    }
+
     if (lastUserMessage.contains("__TEST_ECHO_FORMAT_ENUM")) {
       return Future.successful(structured(format match {
         case EnumFormat(choices) => choices.last
@@ -162,7 +168,7 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid
     } else if (lastUserMessage.contains("__TEST_EMPTY_CONTENT")) {
       // Return empty content (simulates thinking model with no content)
       ""
-    } else if (lastUserMessage.contains("Your choice (one option, no other text):")) {
+    } else if (lastUserMessage.contains("Options:\n") && lastUserMessage.contains("Your choice as {")) {
       // Choose prompt — extract and return the first option from the Options block
       val optionsIdx = lastUserMessage.indexOf("Options:\n")
       if (optionsIdx >= 0) {
diff --git a/tests.txt b/tests.txt
index b2ebac2..b4a1503 100644
--- a/tests.txt
+++ b/tests.txt
@@ -685,6 +685,15 @@ LLMChatWithSchemaSendsSchemaFormat
   llm:get result "name" => "netlogo_schema"
   O> llm:clear-history
 
+LLMChoosePromptMatchesEnumSchema
+  extensions [llm]
+  O> llm:set-api-key "test-key"
+  O> llm:set-provider "openai"
+  O> llm:clear-history
+  llm:choose "__TEST_ECHO_PROMPT_TAIL" ["north" "south"] => ERROR Extension exception: llm:choose: response 'Your choice as {"choice": "<option>"}:' did not match any choice. Choices: north, south
+  llm:history => []
+  O> llm:clear-history
+
 LLMChooseDegradesWhenFormatIgnored
   extensions [llm]
   globals [choice]