From cdbd23335bad1f8264534d94cd3a402773446a22 Mon Sep 17 00:00:00 2001 From: Raphael Southall <69826455+raphasouthall@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:02:04 +0100 Subject: [PATCH] fix(harvest): make the classifier answer every candidate The classifier was dropping most of its input in silence. Against one session's ten candidates the old prompt answered five and kept none, five runs running. The prompt never said every message needs an answer, and it opened by telling the model to skip boilerplate, which pushed it toward blanket refusal. max_tokens of 500 also truncated a ten-line answer mid-summary. Nothing noticed, because only KEEP lines were parsed: an all-SKIP reply, a truncated reply and a reply that never came were the same empty list. State the batch size, demand one line per message in order, reframe the criteria as what to keep and what to skip, and raise the token cap. Parse SKIP lines to count coverage and warn when a batch comes back partly answered. Same ten candidates now answer 10/10 and keep 9, 9, 9, 9, 8 across five runs. Unanswered candidates are still dropped rather than regex-kept: inventing keepers from an answer the model never gave would swap a silent loss for silent noise. Closes #117 --- src/neurostack/harvest.py | 64 ++++++++++++++++++++++++++++----------- tests/test_harvest.py | 62 +++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 17 deletions(-) diff --git a/src/neurostack/harvest.py b/src/neurostack/harvest.py index 4580406..f7a4f63 100644 --- a/src/neurostack/harvest.py +++ b/src/neurostack/harvest.py @@ -680,26 +680,35 @@ def _llm_classify( batch_text = "\n---\n".join(numbered) + # The instruction to answer EVERY message, with the expected line count + # stated, is load-bearing (issue #117). The previous prompt led with the + # keep/skip criteria and offered the two line shapes as alternatives; + # measured against one session's 10 candidates it answered only 5 of + # them and kept 0, five runs in a row. Naming the count and forbidding + # merged or reordered lines took the same 10 to 10 answered and 9 kept, + # stable over five runs. prompt = ( - "You are analyzing an AI coding session transcript. " - "For each numbered message below, decide if it contains a " - "genuinely useful insight worth remembering long-term. " - "Insights include: architectural decisions, bug root causes, " - "tool configurations, user corrections/preferences, " - "discovered facts about infrastructure, and ephemeral " - "session-scoped facts (credentials, endpoints, URLs, " - "current-state notes that go stale quickly).\n\n" - "Skip boilerplate, status updates, and routine tool output.\n\n" + "You are analyzing an AI coding session transcript.\n\n" + f"There are {len(batch)} numbered messages below. Answer with " + f"EXACTLY {len(batch)} lines, one per message, in order, numbered " + f"[1] to [{len(batch)}]. Do not merge, skip or reorder lines. " + "No preamble.\n\n" + "Each line is either:\n" + "[N] KEEP type= summary=\n" + "[N] SKIP\n\n" + "KEEP a message that records any of: an architectural or tooling " + "decision, a bug's root cause or fix, a rule to follow, a " + "discovered fact about a system, a user correction or preference, " + "or a short-lived operational fact such as an endpoint, credential " + "location or current-state note.\n" + "SKIP a message that is only progress narration, a restatement of " + "the task, or raw command output.\n\n" "Type guide: bug=root cause/fix, decision=choice made, " "convention=rule to always follow, learning=discovered fact, " "observation=durable infrastructure fact, " "context=ephemeral/short-lived fact kept only short-term.\n\n" - "For each message, respond with EXACTLY one line:\n" - "[N] KEEP type= " - "summary=\n" - "OR:\n" - "[N] SKIP\n\n" - "Messages:\n" + batch_text + "\n\nAnalysis:" + "Messages:\n" + batch_text + "\n\nAnswer:" ) try: @@ -713,7 +722,9 @@ def _llm_classify( "stream": False, "reasoning_effort": "none", "temperature": 0.1, - "max_tokens": 500, + # One line per candidate, each carrying a summary: 500 + # truncated a 10-message answer mid-line (issue #117). + "max_tokens": 2000, }, timeout=60.0, ) @@ -730,15 +741,24 @@ def _llm_classify( results.append(c) continue + answered: set[int] = set() for line in response.strip().splitlines(): + line = line.strip() + skip = re.match(r"\[(\d+)\]\s+SKIP\b", line) + if skip: + idx = int(skip.group(1)) - 1 + if 0 <= idx < len(batch): + answered.add(idx) + continue m = re.match( r"\[(\d+)\]\s+KEEP\s+type=(\w+)\s+summary=(.+)", - line.strip(), + line, ) if not m: continue idx = int(m.group(1)) - 1 if 0 <= idx < len(batch): + answered.add(idx) c = batch[idx].copy() etype = m.group(2).strip() valid = {"bug", "decision", "convention", "learning", "observation", "context"} @@ -749,6 +769,16 @@ def _llm_classify( c["summary"] = m.group(3).strip() results.append(c) + # A batch the model only partly answered is a silent capture loss: the + # unanswered candidates are dropped, and an all-SKIP reply is otherwise + # indistinguishable from a reply that never arrived (issue #117). Say so. + missing = len(batch) - len(answered) + if missing: + log.warning( + "LLM classify answered %d of %d candidates - %d dropped unclassified", + len(answered), len(batch), missing, + ) + return results diff --git a/tests/test_harvest.py b/tests/test_harvest.py index e1c98f7..6db8469 100644 --- a/tests/test_harvest.py +++ b/tests/test_harvest.py @@ -1,6 +1,7 @@ """Tests for neurostack.harvest — session transcript insight extraction.""" import json +import logging from types import SimpleNamespace from neurostack.harvest import ( @@ -544,6 +545,67 @@ def test_unknown_type_falls_back_to_prefilter(self, monkeypatch): out = _llm_classify(candidates, "http://llm.test", "model") assert out[0]["entity_type"] == "observation" + @staticmethod + def _capture_prompt(monkeypatch, content): + """Same stub, but hand back the prompt the classifier actually sent.""" + import httpx + + sent = {} + + class _Resp: + def raise_for_status(self): + pass + + def json(self): + return {"choices": [{"message": {"content": content}}]} + + def _post(*_a, **kwargs): + sent["prompt"] = kwargs["json"]["messages"][0]["content"] + sent["max_tokens"] = kwargs["json"]["max_tokens"] + return _Resp() + + monkeypatch.setattr(httpx, "post", _post) + cfg = SimpleNamespace(llm_api_key=None) + monkeypatch.setattr("neurostack.config.get_config", lambda: cfg) + monkeypatch.setattr("neurostack.config._auth_headers", lambda _key: {}) + return sent + + @staticmethod + def _candidates(n): + return [{ + "text": f"The root cause of failure number {i} was a missing guard clause.", + "role": "assistant", + "prefilter_type": "bug", + } for i in range(n)] + + def test_prompt_demands_one_line_per_candidate(self, monkeypatch): + # Issue #117: without the explicit count the model answered 5 of 10 and + # kept none. The count instruction is the fix, so it is pinned here. + sent = self._capture_prompt(monkeypatch, "[1] SKIP\n[2] SKIP\n[3] SKIP") + _llm_classify(self._candidates(3), "http://llm.test", "model") + assert "EXACTLY 3 lines" in sent["prompt"] + assert "[1] to [3]" in sent["prompt"] + # One summary-carrying line per candidate does not fit in 500 tokens. + assert sent["max_tokens"] >= 2000 + + def test_partial_answer_is_logged(self, monkeypatch, caplog): + # A dropped candidate is a silent capture loss unless it is announced. + self._stub_llm(monkeypatch, "[1] SKIP\n[2] KEEP type=bug summary=Second one mattered") + with caplog.at_level(logging.WARNING, logger="neurostack"): + out = _llm_classify(self._candidates(5), "http://llm.test", "model") + assert len(out) == 1 + assert "answered 2 of 5" in caplog.text + assert "3 dropped unclassified" in caplog.text + + def test_full_answer_logs_nothing(self, monkeypatch, caplog): + # An all-SKIP reply that covers every candidate is a real verdict, not a + # failure — it must not cry wolf. + self._stub_llm(monkeypatch, "[1] SKIP\n[2] SKIP\n[3] SKIP") + with caplog.at_level(logging.WARNING, logger="neurostack"): + out = _llm_classify(self._candidates(3), "http://llm.test", "model") + assert out == [] + assert "dropped unclassified" not in caplog.text + # --------------------------------------------------------------------------- # harvest_sessions — per-type TTL on harvest-created memories (issue #36)