Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions src/main/LLMExtension.scala
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,10 @@ class LLMExtension extends DefaultClassManager {
}

object ChooseReporter extends Reporter {
/** Final line of the choose prompt; names the shape the enum schema enforces. */
val ChoiceInstruction: String =
s"""Your choice as {"${EnumFormat.ChoiceKey}": "<option>"}:"""

override def getSyntax: Syntax = Syntax.reporterSyntax(
right = List(Syntax.StringType, Syntax.ListType),
ret = Syntax.StringType
Expand All @@ -613,14 +617,18 @@ class LLMExtension extends DefaultClassManager {
throw new ExtensionException("Choice list cannot be empty")
}

// The wording must describe the same shape as the EnumFormat schema
// sent with the request. A provider that validates the reply against
// the schema server-side (Groq does) rejects the whole response with
// HTTP 400 when the model follows a prompt asking for bare text.
val systemPrompt = "You are a decision-making assistant. " +
"When given options, respond with EXACTLY one option from the list. " +
"Rules: Reply with ONLY the option text. " +
"No explanation, no numbering, no punctuation, no quotes, no extra words. " +
"Copy the option exactly as written."
"When given options, choose EXACTLY one option from the list. " +
s"""Reply with a JSON object of the form {"${EnumFormat.ChoiceKey}": "<option>"} """ +
"where <option> is copied exactly as written. " +
"No explanation, no extra keys, no extra text."

val userPrompt = s"$prompt\n\nOptions:\n${choices.mkString("\n")}\n\n" +
"Your choice (one option, no other text):"
ChooseReporter.ChoiceInstruction

// Build temp history with system prompt — don't mutate permanent history
val tempHistory = ArrayBuffer.from(snapshotHistory(agent))
Expand Down
8 changes: 7 additions & 1 deletion src/test/DeterministicTestProvider.scala
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid
// primitive that stopped constraining its request fails a test instead of
// silently degrading. The enum echo answers with the LAST choice because
// the unconstrained prompt-only path answers with the first.
// __TEST_ECHO_PROMPT_TAIL answers with the prompt's final line so a test can
// pin the instruction the model actually sees next to the schema it is sent.
if (lastUserMessage.contains("__TEST_ECHO_PROMPT_TAIL")) {
return Future.successful(structured(lastUserMessage.linesIterator.toSeq.last))
}

if (lastUserMessage.contains("__TEST_ECHO_FORMAT_ENUM")) {
return Future.successful(structured(format match {
case EnumFormat(choices) => choices.last
Expand Down Expand Up @@ -162,7 +168,7 @@ class DeterministicTestProvider(implicit ec: ExecutionContext) extends LLMProvid
} else if (lastUserMessage.contains("__TEST_EMPTY_CONTENT")) {
// Return empty content (simulates thinking model with no content)
""
} else if (lastUserMessage.contains("Your choice (one option, no other text):")) {
} else if (lastUserMessage.contains("Options:\n") && lastUserMessage.contains("Your choice as {")) {
// Choose prompt — extract and return the first option from the Options block
val optionsIdx = lastUserMessage.indexOf("Options:\n")
if (optionsIdx >= 0) {
Expand Down
9 changes: 9 additions & 0 deletions tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -685,6 +685,15 @@ LLMChatWithSchemaSendsSchemaFormat
llm:get result "name" => "netlogo_schema"
O> llm:clear-history

LLMChoosePromptMatchesEnumSchema
extensions [llm]
O> llm:set-api-key "test-key"
O> llm:set-provider "openai"
O> llm:clear-history
llm:choose "__TEST_ECHO_PROMPT_TAIL" ["north" "south"] => ERROR Extension exception: llm:choose: response 'Your choice as {"choice": "<option>"}:' did not match any choice. Choices: north, south
llm:history => []
O> llm:clear-history

LLMChooseDegradesWhenFormatIgnored
extensions [llm]
globals [choice]
Expand Down
Loading