fix(local-llm): managed Qwen 3.5 stalls in an unterminated tool-call grammar

grammars/tool-call.gbnf

@@ -43,4 +43,13 @@ exponent ::= ("e" | "E") ("+" | "-")? [0-9]+
 boolean ::= "true" | "false"
 null-lit ::= "null"
 
-ws ::= [ \t\n\r]*
+# Bounded, deliberately. Whitespace is legal at ~10 seams inside the JSON
+# body and GBNF masks EOG until the whole root is satisfied, so an
+# unbounded `[ \t\n\r]*` gives a stuck sampler an infinite legal move: it
+# emits newlines until `n_predict` while the stream parser (still waiting
+# on `"tool":`) surfaces nothing to the UI — a silent multi-minute hang.
+# 64 is far past any realistic newline+indent run (deep pretty-printed
+# JSON tops out well under it); past that the mask simply forces the next
+# real token, which is the desired recovery. Same grouped `{0,n}` shape
+# already used for the reasoning-seam whitespace rule in build-grammar.ts.
+ws ::= ( [ \t\n\r] ){0,64}

src/llm/grammar/build-grammar.test.ts

@@ -15,6 +15,24 @@ describe("buildGrammar", () => {
     expect(grammar).not.toContain("think-prelude ::= think-body");
   });
 
+  // GBNF masks EOG until the whole root is satisfied, so any unbounded
+  // rule is an infinite legal move for a stuck sampler. `ws` appears at
+  // ~10 seams inside the JSON body; unbounded it produced a silent
+  // newline-until-n_predict hang that surfaced nothing to the UI.
+  it("bounds the shared whitespace rule so it cannot loop forever", async () => {
+    for (const profile of [
+      PLAIN_INSTRUCT_PROFILE,
+      QWEN_THINK_PROFILE,
+      GEMMA4_THINK_PROFILE,
+    ]) {
+      const grammar = await buildGrammar(profile);
+      const ws = grammar.match(/^ws ::= .*$/m);
+      expect(ws, profile.id).not.toBeNull();
+      expect(ws![0], profile.id).toMatch(/\{0,\d+\}\s*$/);
+      expect(ws![0], profile.id).not.toMatch(/\*\s*$/);
+    }
+  });
+
   it("builds a qwen think grammar with a think prelude routed into the array", async () => {
     const grammar = await buildGrammar(QWEN_THINK_PROFILE);
     expect(grammar).toContain("root ::= think-prelude tool-call-array");

src/llm/index.ts

@@ -9,10 +9,11 @@ export type {
 } from "./llama-server-client.js";
 export { checkLlamaServer } from "./llama-server-health.js";
 export type { HealthCheckOptions, HealthResult } from "./llama-server-health.js";
-export { SlotManager, hashPrefix } from "./slot-manager.js";
+export { SlotManager, hashPrefix, DEFAULT_SLOT_COUNT } from "./slot-manager.js";
 export type { SlotAssignment } from "./slot-manager.js";
 export {
   detectModelProfile,
+  extractTotalSlots,
   GEMMA4_THINK_PROFILE,
   PLAIN_INSTRUCT_PROFILE,
   QWEN_THINK_PROFILE,

src/llm/llama-server-client.test.ts

@@ -159,6 +159,55 @@ describe("LlamaServerClient.complete", () => {
     expect(result.content).toBe("ok");
   });
 
+  // A slow model that blows `requestTimeoutMs` surfaces as an abort with
+  // `status === null`, structurally identical to a dropped socket. Retrying
+  // it burns another full timeout of GPU time per attempt and cannot
+  // succeed, so it must short-circuit like a 4xx does.
+  it("does not retry when its own requestTimeoutMs fires", async () => {
+    let calls = 0;
+    const client = new LlamaServerClient({
+      baseUrl: "http://127.0.0.1:9999",
+      requestTimeoutMs: 5,
+      fetchImpl: createMockFetch(
+        (_url, init) =>
+          new Promise((_resolve, reject) => {
+            calls += 1;
+            init.signal?.addEventListener("abort", () => {
+              reject(Object.assign(new Error("aborted"), { name: "AbortError" }));
+            });
+          }),
+      ),
+      completionRetries: 5,
+      completionRetryBackoffMs: 0,
+      sleep: async () => {},
+    });
+    await expect(client.complete({ prompt: "hi" })).rejects.toMatchObject({
+      name: "LlamaServerError",
+      status: null,
+      timedOut: true,
+    });
+    expect(calls).toBe(1);
+  });
+
+  it("still retries genuine transport failures", async () => {
+    let calls = 0;
+    const client = new LlamaServerClient({
+      baseUrl: "http://127.0.0.1:9999",
+      requestTimeoutMs: 60_000,
+      fetchImpl: createMockFetch(async () => {
+        calls += 1;
+        throw new Error("ECONNRESET");
+      }),
+      completionRetries: 3,
+      completionRetryBackoffMs: 0,
+      sleep: async () => {},
+    });
+    await expect(client.complete({ prompt: "hi" })).rejects.toMatchObject({
+      timedOut: false,
+    });
+    expect(calls).toBe(3);
+  });
+
   it("does not retry on 4xx grammar/validation errors", async () => {
     let calls = 0;
     const client = new LlamaServerClient({

src/llm/llama-server-client.ts

@@ -42,6 +42,15 @@ export class LlamaServerError extends Error {
     message: string,
     public readonly status: number | null,
     public readonly url: string,
+    /**
+     * True when *our own* `requestTimeoutMs` controller fired rather than
+     * the transport failing. Both surface as `status === null`, but a
+     * timeout is a "the model is slower than the budget" signal, not a
+     * transient blip — replaying it just burns another full timeout of
+     * GPU time (3 attempts x 300s = 15 silent minutes). See
+     * `isRetryableLlamaError`.
+     */
+    public readonly timedOut = false,
   ) {
     super(message);
     this.name = "LlamaServerError";
@@ -193,7 +202,7 @@ export class LlamaServerClient {
     return this.runWithRetry(
       url,
       async () => {
-        const { controller, cleanup } = this.createRequestController(
+        const { controller, cleanup, timedOut } = this.createRequestController(
           request.signal,
         );
         try {
@@ -209,9 +218,7 @@ export class LlamaServerClient {
           const json = (await response.json()) as Record<string, unknown>;
           return normaliseCompletionResponse(json);
         } catch (err) {
-          if (err instanceof LlamaServerError) throw err;
-          const message = err instanceof Error ? err.message : String(err);
-          throw new LlamaServerError(message, null, url);
+          throw this.wrapTransportError(err, url, timedOut());
         } finally {
           cleanup();
         }
@@ -231,14 +238,14 @@ export class LlamaServerClient {
       response: Response;
       controller: AbortController;
       cleanup: () => void;
+      timedOut: () => boolean;
     };
     try {
       opened = await this.runWithRetry(
         url,
         async () => {
-          const { controller, cleanup } = this.createRequestController(
-            request.signal,
-          );
+          const { controller, cleanup, timedOut } =
+            this.createRequestController(request.signal);
           try {
             const response = await this.fetchImpl(url, {
               method: "POST",
@@ -249,12 +256,10 @@ export class LlamaServerClient {
             if (!response.ok || !response.body) {
               throw await buildHttpError(response, url);
             }
-            return { response, controller, cleanup };
+            return { response, controller, cleanup, timedOut };
           } catch (err) {
             cleanup();
-            if (err instanceof LlamaServerError) throw err;
-            const message = err instanceof Error ? err.message : String(err);
-            throw new LlamaServerError(message, null, url);
+            throw this.wrapTransportError(err, url, timedOut());
           }
         },
         request.signal,
@@ -264,7 +269,7 @@ export class LlamaServerClient {
       const message = err instanceof Error ? err.message : String(err);
       throw new LlamaServerError(message, null, url);
     }
-    const { response, cleanup } = opened;
+    const { response, cleanup, timedOut } = opened;
     let finalResult: CompletionResult = {
       content: "",
       reasoningContent: "",
@@ -331,9 +336,7 @@ export class LlamaServerClient {
       }
       return finalResult;
     } catch (err) {
-      if (err instanceof LlamaServerError) throw err;
-      const message = err instanceof Error ? err.message : String(err);
-      throw new LlamaServerError(message, null, url);
+      throw this.wrapTransportError(err, url, timedOut());
     } finally {
       cleanup();
     }
@@ -350,27 +353,59 @@ export class LlamaServerClient {
   private createRequestController(externalSignal?: AbortSignal): {
     controller: AbortController;
     cleanup: () => void;
+    /** True once the per-request timeout (not the caller) fired the abort. */
+    timedOut: () => boolean;
   } {
     const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), this.requestTimeoutMs);
+    let expired = false;
+    const timer = setTimeout(() => {
+      expired = true;
+      controller.abort();
+    }, this.requestTimeoutMs);
+    const timedOut = (): boolean => expired;
     if (!externalSignal) {
-      return { controller, cleanup: () => clearTimeout(timer) };
+      return { controller, cleanup: () => clearTimeout(timer), timedOut };
     }
     if (externalSignal.aborted) {
       controller.abort();
-      return { controller, cleanup: () => clearTimeout(timer) };
+      return { controller, cleanup: () => clearTimeout(timer), timedOut };
     }
     const onAbort = (): void => controller.abort();
     externalSignal.addEventListener("abort", onAbort, { once: true });
     return {
       controller,
+      timedOut,
       cleanup: () => {
         clearTimeout(timer);
         externalSignal.removeEventListener("abort", onAbort);
       },
     };
   }
 
+  /**
+   * Normalise a caught transport failure into a `LlamaServerError`,
+   * preserving whether it was our own request-timeout so the retry policy
+   * can refuse to replay it. Pass-through for errors already of that type.
+   */
+  private wrapTransportError(
+    err: unknown,
+    url: string,
+    timedOut: boolean,
+  ): LlamaServerError {
+    if (err instanceof LlamaServerError) return err;
+    if (timedOut) {
+      return new LlamaServerError(
+        `llama-server request exceeded requestTimeoutMs (${this.requestTimeoutMs}ms) — ` +
+          `raise localModels.requestTimeoutMs or lower completionMaxTokens`,
+        null,
+        url,
+        true,
+      );
+    }
+    const message = err instanceof Error ? err.message : String(err);
+    return new LlamaServerError(message, null, url);
+  }
+
   private prepareRequest(
     request: CompletionRequest,
     stream: boolean,
@@ -528,6 +563,10 @@ function toNumber(value: unknown, fallback = 0): number {
  */
 function isRetryableLlamaError(err: unknown): boolean {
   if (!(err instanceof LlamaServerError)) return false;
+  // Our own request-timeout, not a transport failure. Replaying it costs
+  // another full `requestTimeoutMs` of GPU time and cannot succeed if the
+  // model simply needs longer than the budget.
+  if (err.timedOut) return false;
   if (err.status === null) return true;
   if (err.status >= 500 && err.status < 600) return true;
   return false;

src/llm/model-profile-manager.test.ts

@@ -34,6 +34,56 @@ function makeLlamaStub(
   } as { client: Pick<LlamaServerClient, "fetchProps">; calls: number };
 }
 
+describe("ModelProfileManager slot discovery", () => {
+  // Managed mode defers the boot health check, so bootstrap builds the
+  // SlotManager on a one-slot default. This refresh hook is the only path
+  // that ever widens it to the daemon's real `--parallel` count.
+  it("reports /props.total_slots on a successful refresh", async () => {
+    const stub = makeLlamaStub([{ ...QWEN3_PROPS, total_slots: 2 }]);
+    const seen: number[] = [];
+    const mgr = new ModelProfileManager({
+      llama: stub.client as LlamaServerClient,
+      initialProfile: PLAIN_INSTRUCT_PROFILE,
+      initialGrammar: await buildGrammar(PLAIN_INSTRUCT_PROFILE),
+      initialModelId: null,
+      onTotalSlots: (n) => seen.push(n),
+    });
+    await mgr.refresh();
+    expect(seen).toEqual([2]);
+  });
+
+  it("stays silent when the server omits total_slots", async () => {
+    const stub = makeLlamaStub([QWEN3_PROPS]);
+    const seen: number[] = [];
+    const mgr = new ModelProfileManager({
+      llama: stub.client as LlamaServerClient,
+      initialProfile: PLAIN_INSTRUCT_PROFILE,
+      initialGrammar: await buildGrammar(PLAIN_INSTRUCT_PROFILE),
+      initialModelId: null,
+      onTotalSlots: (n) => seen.push(n),
+    });
+    await mgr.refresh();
+    expect(seen).toEqual([]);
+  });
+
+  it("reports slots even when the probe body fails profile detection", async () => {
+    // Slot discovery must not be hostage to the grammar rebuild path — an
+    // oversized pool thrashes the KV cache on every turn regardless of
+    // whether the profile changed.
+    const stub = makeLlamaStub([{ total_slots: 4 }]);
+    const seen: number[] = [];
+    const mgr = new ModelProfileManager({
+      llama: stub.client as LlamaServerClient,
+      initialProfile: PLAIN_INSTRUCT_PROFILE,
+      initialGrammar: await buildGrammar(PLAIN_INSTRUCT_PROFILE),
+      initialModelId: null,
+      onTotalSlots: (n) => seen.push(n),
+    });
+    await mgr.refresh();
+    expect(seen).toEqual([4]);
+  });
+});
+
 describe("ModelProfileManager", () => {
   it("reports the initial profile/grammar until a mismatch is observed", async () => {
     const grammar = await buildGrammar(GEMMA4_THINK_PROFILE);