Make the harvest classifier answer every candidate

src/neurostack/harvest.py

@@ -680,26 +680,35 @@ def _llm_classify(
 
         batch_text = "\n---\n".join(numbered)
 
+        # The instruction to answer EVERY message, with the expected line count
+        # stated, is load-bearing (issue #117). The previous prompt led with the
+        # keep/skip criteria and offered the two line shapes as alternatives;
+        # measured against one session's 10 candidates it answered only 5 of
+        # them and kept 0, five runs in a row. Naming the count and forbidding
+        # merged or reordered lines took the same 10 to 10 answered and 9 kept,
+        # stable over five runs.
         prompt = (
-            "You are analyzing an AI coding session transcript. "
-            "For each numbered message below, decide if it contains a "
-            "genuinely useful insight worth remembering long-term. "
-            "Insights include: architectural decisions, bug root causes, "
-            "tool configurations, user corrections/preferences, "
-            "discovered facts about infrastructure, and ephemeral "
-            "session-scoped facts (credentials, endpoints, URLs, "
-            "current-state notes that go stale quickly).\n\n"
-            "Skip boilerplate, status updates, and routine tool output.\n\n"
+            "You are analyzing an AI coding session transcript.\n\n"
+            f"There are {len(batch)} numbered messages below. Answer with "
+            f"EXACTLY {len(batch)} lines, one per message, in order, numbered "
+            f"[1] to [{len(batch)}]. Do not merge, skip or reorder lines. "
+            "No preamble.\n\n"
+            "Each line is either:\n"
+            "[N] KEEP type=<bug|decision|convention|learning|observation|"
+            "context> summary=<one sentence>\n"
+            "[N] SKIP\n\n"
+            "KEEP a message that records any of: an architectural or tooling "
+            "decision, a bug's root cause or fix, a rule to follow, a "
+            "discovered fact about a system, a user correction or preference, "
+            "or a short-lived operational fact such as an endpoint, credential "
+            "location or current-state note.\n"
+            "SKIP a message that is only progress narration, a restatement of "
+            "the task, or raw command output.\n\n"
             "Type guide: bug=root cause/fix, decision=choice made, "
             "convention=rule to always follow, learning=discovered fact, "
             "observation=durable infrastructure fact, "
             "context=ephemeral/short-lived fact kept only short-term.\n\n"
-            "For each message, respond with EXACTLY one line:\n"
-            "[N] KEEP type=<bug|decision|convention|learning|observation|context> "
-            "summary=<one sentence summary>\n"
-            "OR:\n"
-            "[N] SKIP\n\n"
-            "Messages:\n" + batch_text + "\n\nAnalysis:"
+            "Messages:\n" + batch_text + "\n\nAnswer:"
         )
 
         try:
@@ -713,7 +722,9 @@ def _llm_classify(
                     "stream": False,
                     "reasoning_effort": "none",
                     "temperature": 0.1,
-                    "max_tokens": 500,
+                    # One line per candidate, each carrying a summary: 500
+                    # truncated a 10-message answer mid-line (issue #117).
+                    "max_tokens": 2000,
                 },
                 timeout=60.0,
             )
@@ -730,15 +741,24 @@ def _llm_classify(
                 results.append(c)
             continue
 
+        answered: set[int] = set()
         for line in response.strip().splitlines():
+            line = line.strip()
+            skip = re.match(r"\[(\d+)\]\s+SKIP\b", line)
+            if skip:
+                idx = int(skip.group(1)) - 1
+                if 0 <= idx < len(batch):
+                    answered.add(idx)
+                continue
             m = re.match(
                 r"\[(\d+)\]\s+KEEP\s+type=(\w+)\s+summary=(.+)",
-                line.strip(),
+                line,
             )
             if not m:
                 continue
             idx = int(m.group(1)) - 1
             if 0 <= idx < len(batch):
+                answered.add(idx)
                 c = batch[idx].copy()
                 etype = m.group(2).strip()
                 valid = {"bug", "decision", "convention", "learning", "observation", "context"}
@@ -749,6 +769,16 @@ def _llm_classify(
                 c["summary"] = m.group(3).strip()
                 results.append(c)
 
+        # A batch the model only partly answered is a silent capture loss: the
+        # unanswered candidates are dropped, and an all-SKIP reply is otherwise
+        # indistinguishable from a reply that never arrived (issue #117). Say so.
+        missing = len(batch) - len(answered)
+        if missing:
+            log.warning(
+                "LLM classify answered %d of %d candidates - %d dropped unclassified",
+                len(answered), len(batch), missing,
+            )
+
     return results
 
 

tests/test_harvest.py

@@ -1,6 +1,7 @@
 """Tests for neurostack.harvest — session transcript insight extraction."""
 
 import json
+import logging
 from types import SimpleNamespace
 
 from neurostack.harvest import (
@@ -544,6 +545,67 @@ def test_unknown_type_falls_back_to_prefilter(self, monkeypatch):
         out = _llm_classify(candidates, "http://llm.test", "model")
         assert out[0]["entity_type"] == "observation"
 
+    @staticmethod
+    def _capture_prompt(monkeypatch, content):
+        """Same stub, but hand back the prompt the classifier actually sent."""
+        import httpx
+
+        sent = {}
+
+        class _Resp:
+            def raise_for_status(self):
+                pass
+
+            def json(self):
+                return {"choices": [{"message": {"content": content}}]}
+
+        def _post(*_a, **kwargs):
+            sent["prompt"] = kwargs["json"]["messages"][0]["content"]
+            sent["max_tokens"] = kwargs["json"]["max_tokens"]
+            return _Resp()
+
+        monkeypatch.setattr(httpx, "post", _post)
+        cfg = SimpleNamespace(llm_api_key=None)
+        monkeypatch.setattr("neurostack.config.get_config", lambda: cfg)
+        monkeypatch.setattr("neurostack.config._auth_headers", lambda _key: {})
+        return sent
+
+    @staticmethod
+    def _candidates(n):
+        return [{
+            "text": f"The root cause of failure number {i} was a missing guard clause.",
+            "role": "assistant",
+            "prefilter_type": "bug",
+        } for i in range(n)]
+
+    def test_prompt_demands_one_line_per_candidate(self, monkeypatch):
+        # Issue #117: without the explicit count the model answered 5 of 10 and
+        # kept none. The count instruction is the fix, so it is pinned here.
+        sent = self._capture_prompt(monkeypatch, "[1] SKIP\n[2] SKIP\n[3] SKIP")
+        _llm_classify(self._candidates(3), "http://llm.test", "model")
+        assert "EXACTLY 3 lines" in sent["prompt"]
+        assert "[1] to [3]" in sent["prompt"]
+        # One summary-carrying line per candidate does not fit in 500 tokens.
+        assert sent["max_tokens"] >= 2000
+
+    def test_partial_answer_is_logged(self, monkeypatch, caplog):
+        # A dropped candidate is a silent capture loss unless it is announced.
+        self._stub_llm(monkeypatch, "[1] SKIP\n[2] KEEP type=bug summary=Second one mattered")
+        with caplog.at_level(logging.WARNING, logger="neurostack"):
+            out = _llm_classify(self._candidates(5), "http://llm.test", "model")
+        assert len(out) == 1
+        assert "answered 2 of 5" in caplog.text
+        assert "3 dropped unclassified" in caplog.text
+
+    def test_full_answer_logs_nothing(self, monkeypatch, caplog):
+        # An all-SKIP reply that covers every candidate is a real verdict, not a
+        # failure — it must not cry wolf.
+        self._stub_llm(monkeypatch, "[1] SKIP\n[2] SKIP\n[3] SKIP")
+        with caplog.at_level(logging.WARNING, logger="neurostack"):
+            out = _llm_classify(self._candidates(3), "http://llm.test", "model")
+        assert out == []
+        assert "dropped unclassified" not in caplog.text
+
 
 # ---------------------------------------------------------------------------
 # harvest_sessions — per-type TTL on harvest-created memories (issue #36)