diff --git a/src/neurostack/harvest.py b/src/neurostack/harvest.py index 4580406..f7a4f63 100644 --- a/src/neurostack/harvest.py +++ b/src/neurostack/harvest.py @@ -680,26 +680,35 @@ def _llm_classify( batch_text = "\n---\n".join(numbered) + # The instruction to answer EVERY message, with the expected line count + # stated, is load-bearing (issue #117). The previous prompt led with the + # keep/skip criteria and offered the two line shapes as alternatives; + # measured against one session's 10 candidates it answered only 5 of + # them and kept 0, five runs in a row. Naming the count and forbidding + # merged or reordered lines took the same 10 to 10 answered and 9 kept, + # stable over five runs. prompt = ( - "You are analyzing an AI coding session transcript. " - "For each numbered message below, decide if it contains a " - "genuinely useful insight worth remembering long-term. " - "Insights include: architectural decisions, bug root causes, " - "tool configurations, user corrections/preferences, " - "discovered facts about infrastructure, and ephemeral " - "session-scoped facts (credentials, endpoints, URLs, " - "current-state notes that go stale quickly).\n\n" - "Skip boilerplate, status updates, and routine tool output.\n\n" + "You are analyzing an AI coding session transcript.\n\n" + f"There are {len(batch)} numbered messages below. Answer with " + f"EXACTLY {len(batch)} lines, one per message, in order, numbered " + f"[1] to [{len(batch)}]. Do not merge, skip or reorder lines. " + "No preamble.\n\n" + "Each line is either:\n" + "[N] KEEP type= summary=\n" + "[N] SKIP\n\n" + "KEEP a message that records any of: an architectural or tooling " + "decision, a bug's root cause or fix, a rule to follow, a " + "discovered fact about a system, a user correction or preference, " + "or a short-lived operational fact such as an endpoint, credential " + "location or current-state note.\n" + "SKIP a message that is only progress narration, a restatement of " + "the task, or raw command output.\n\n" "Type guide: bug=root cause/fix, decision=choice made, " "convention=rule to always follow, learning=discovered fact, " "observation=durable infrastructure fact, " "context=ephemeral/short-lived fact kept only short-term.\n\n" - "For each message, respond with EXACTLY one line:\n" - "[N] KEEP type= " - "summary=\n" - "OR:\n" - "[N] SKIP\n\n" - "Messages:\n" + batch_text + "\n\nAnalysis:" + "Messages:\n" + batch_text + "\n\nAnswer:" ) try: @@ -713,7 +722,9 @@ def _llm_classify( "stream": False, "reasoning_effort": "none", "temperature": 0.1, - "max_tokens": 500, + # One line per candidate, each carrying a summary: 500 + # truncated a 10-message answer mid-line (issue #117). + "max_tokens": 2000, }, timeout=60.0, ) @@ -730,15 +741,24 @@ def _llm_classify( results.append(c) continue + answered: set[int] = set() for line in response.strip().splitlines(): + line = line.strip() + skip = re.match(r"\[(\d+)\]\s+SKIP\b", line) + if skip: + idx = int(skip.group(1)) - 1 + if 0 <= idx < len(batch): + answered.add(idx) + continue m = re.match( r"\[(\d+)\]\s+KEEP\s+type=(\w+)\s+summary=(.+)", - line.strip(), + line, ) if not m: continue idx = int(m.group(1)) - 1 if 0 <= idx < len(batch): + answered.add(idx) c = batch[idx].copy() etype = m.group(2).strip() valid = {"bug", "decision", "convention", "learning", "observation", "context"} @@ -749,6 +769,16 @@ def _llm_classify( c["summary"] = m.group(3).strip() results.append(c) + # A batch the model only partly answered is a silent capture loss: the + # unanswered candidates are dropped, and an all-SKIP reply is otherwise + # indistinguishable from a reply that never arrived (issue #117). Say so. + missing = len(batch) - len(answered) + if missing: + log.warning( + "LLM classify answered %d of %d candidates - %d dropped unclassified", + len(answered), len(batch), missing, + ) + return results diff --git a/tests/test_harvest.py b/tests/test_harvest.py index e1c98f7..6db8469 100644 --- a/tests/test_harvest.py +++ b/tests/test_harvest.py @@ -1,6 +1,7 @@ """Tests for neurostack.harvest — session transcript insight extraction.""" import json +import logging from types import SimpleNamespace from neurostack.harvest import ( @@ -544,6 +545,67 @@ def test_unknown_type_falls_back_to_prefilter(self, monkeypatch): out = _llm_classify(candidates, "http://llm.test", "model") assert out[0]["entity_type"] == "observation" + @staticmethod + def _capture_prompt(monkeypatch, content): + """Same stub, but hand back the prompt the classifier actually sent.""" + import httpx + + sent = {} + + class _Resp: + def raise_for_status(self): + pass + + def json(self): + return {"choices": [{"message": {"content": content}}]} + + def _post(*_a, **kwargs): + sent["prompt"] = kwargs["json"]["messages"][0]["content"] + sent["max_tokens"] = kwargs["json"]["max_tokens"] + return _Resp() + + monkeypatch.setattr(httpx, "post", _post) + cfg = SimpleNamespace(llm_api_key=None) + monkeypatch.setattr("neurostack.config.get_config", lambda: cfg) + monkeypatch.setattr("neurostack.config._auth_headers", lambda _key: {}) + return sent + + @staticmethod + def _candidates(n): + return [{ + "text": f"The root cause of failure number {i} was a missing guard clause.", + "role": "assistant", + "prefilter_type": "bug", + } for i in range(n)] + + def test_prompt_demands_one_line_per_candidate(self, monkeypatch): + # Issue #117: without the explicit count the model answered 5 of 10 and + # kept none. The count instruction is the fix, so it is pinned here. + sent = self._capture_prompt(monkeypatch, "[1] SKIP\n[2] SKIP\n[3] SKIP") + _llm_classify(self._candidates(3), "http://llm.test", "model") + assert "EXACTLY 3 lines" in sent["prompt"] + assert "[1] to [3]" in sent["prompt"] + # One summary-carrying line per candidate does not fit in 500 tokens. + assert sent["max_tokens"] >= 2000 + + def test_partial_answer_is_logged(self, monkeypatch, caplog): + # A dropped candidate is a silent capture loss unless it is announced. + self._stub_llm(monkeypatch, "[1] SKIP\n[2] KEEP type=bug summary=Second one mattered") + with caplog.at_level(logging.WARNING, logger="neurostack"): + out = _llm_classify(self._candidates(5), "http://llm.test", "model") + assert len(out) == 1 + assert "answered 2 of 5" in caplog.text + assert "3 dropped unclassified" in caplog.text + + def test_full_answer_logs_nothing(self, monkeypatch, caplog): + # An all-SKIP reply that covers every candidate is a real verdict, not a + # failure — it must not cry wolf. + self._stub_llm(monkeypatch, "[1] SKIP\n[2] SKIP\n[3] SKIP") + with caplog.at_level(logging.WARNING, logger="neurostack"): + out = _llm_classify(self._candidates(3), "http://llm.test", "model") + assert out == [] + assert "dropped unclassified" not in caplog.text + # --------------------------------------------------------------------------- # harvest_sessions — per-type TTL on harvest-created memories (issue #36)