From d61de5ea07e7749ac8049e713a1079e9c002aa83 Mon Sep 17 00:00:00 2001 From: dat999zx Date: Wed, 19 Aug 2026 19:02:39 +0700 Subject: [PATCH] feat: add knowl and agentmemory as memory methods Two agentic-memory systems that the paper does not currently evaluate, wired in the same way as the existing memory agents so their rows sit beside the published ones on equal terms. WHY THESE TWO. Both resolve write-time conflicts, which is the behaviour FactConsolidation isolates, and neither has an FC score anywhere. agentmemory publishes LongMemEval-S retrieval recall, a different task on a different metric; knowl publishes nothing outside this harness. CONFIGS ARE COPIED FROM Simple_rag_bm25, NOT CHOSEN. retrieve_num 10, temperature 0.7, input_length_limit 10000000, buffer_length 200 -- the same values BM25, Zep, Cognee, HippoRAG-v2, RAPTOR, GraphRAG, Self-RAG and every embedding baseline already use. The reader path is shared between both methods and reproduces _handle_bm25_rag exactly: each retrieved item gets a trailing newline, items are labelled "Memory i:" and joined, and the instruction trails the facts under the generic system template. Retrieval goes through _extract_retrieval_query, as the RAG handlers do -- skipping it is not a small error, since the ~200 tokens of task boilerplate are byte-identical across all 100 questions and retrieving on them retrieves on noise. Neither method installs anything into this venv. knowl runs as a Node subprocess over a sentinel-framed stdio protocol (the embedding runtime and SQLite bindings both write to stdout freely, and an unframed protocol eventually swallows a log line and desynchronises into a plausible score rather than an error). agentmemory runs as its own service and is reached over REST. Both ingest an identical parsed fact list, one record per write, in context order. Feeding raw chunks was rejected: agentmemory stores one memory per remember call, so a 4096-char chunk becomes a single record holding ~70 facts, which measures chunking rather than memory. Measured on this harness, FC-SH SubEM, gpt-4o-mini reader: 6k 262k knowl 95.0 89.0 agentmemory 83.0 79.0 Single runs at temperature 0.7; treat as directional. One asymmetry noticed while matching the reader: _handle_mem0_agent places its memories in a system message and appends a "Current Time:" line, where the RAG family does neither. Both methods here follow the RAG family. Flagging it in case cross-family comparisons are meant to hold that constant. --- agent.py | 14 +- .../gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml | 11 + .../Knowl_gpt-4o-mini-nosupersede.yaml | 16 ++ .../gpt-4o-mini/Knowl_gpt-4o-mini.yaml | 16 ++ methods/agentmemory.py | 190 ++++++++++++++++ methods/knowl.py | 213 ++++++++++++++++++ 6 files changed, 459 insertions(+), 1 deletion(-) create mode 100644 configs/agent_conf/RAG_Agents/gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml create mode 100644 configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini-nosupersede.yaml create mode 100644 configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini.yaml create mode 100644 methods/agentmemory.py create mode 100644 methods/knowl.py diff --git a/agent.py b/agent.py index d3cef2dd..ae859fb5 100644 --- a/agent.py +++ b/agent.py @@ -74,6 +74,12 @@ def _initialize_agent_by_type(self, agent_config, dataset_config): self._initialize_cognee_agent(agent_config, dataset_config) elif self._is_agent_type("zep"): self._initialize_zep_agent(agent_config) + elif self._is_agent_type("knowl"): + from methods.knowl import initialize_knowl_agent + initialize_knowl_agent(self, agent_config) + elif self._is_agent_type("agentmemory"): + from methods.agentmemory import initialize_agentmemory_agent + initialize_agentmemory_agent(self, agent_config) elif self._is_agent_type("rag"): self._initialize_rag_agent(agent_config, dataset_config) else: @@ -271,7 +277,7 @@ def send_message(self, message, memorizing=False, query_id=None, context_id=None # Route to appropriate agent handler based on agent type if 'Long_context_agent' in self.agent_name: return self._handle_long_context_agent(message, memorizing) - elif any(self._is_agent_type(agent_type) for agent_type in ["letta", "cognee", "mem0", "zep"]): + elif any(self._is_agent_type(agent_type) for agent_type in ["letta", "cognee", "mem0", "zep", "knowl", "agentmemory"]): return self._handle_memory_agent(message, memorizing, query_id, context_id) elif self._is_agent_type("rag"): return self._handle_rag_agent(message, memorizing, query_id, context_id) @@ -406,6 +412,12 @@ def _handle_memory_agent(self, message, memorizing, query_id, context_id): return self._handle_mem0_agent(message, memorizing, query_id, context_id) elif self._is_agent_type("zep"): return self._handle_zep_agent(message, memorizing, query_id, context_id) + elif self._is_agent_type("knowl"): + from methods.knowl import handle_knowl_agent + return handle_knowl_agent(self, message, memorizing, query_id, context_id) + elif self._is_agent_type("agentmemory"): + from methods.agentmemory import handle_agentmemory_agent + return handle_agentmemory_agent(self, message, memorizing, query_id, context_id) else: raise NotImplementedError(f"Memory agent type not supported: {self.agent_name}") diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml new file mode 100644 index 00000000..ca15c9a1 --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml @@ -0,0 +1,11 @@ +# agentmemory v0.9.29, driven over REST. Every value except the output_dir is copied from +# Simple_rag_bm25 so the row sits beside the published baselines on the same terms: +# retrieve_num 10 is what BM25, Zep, Cognee, HippoRAG-v2 and the embedding baselines use. +agent_name: Agentic_memory_agentmemory +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/agentmemory-gpt-4o-mini + +retrieve_num: 10 diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini-nosupersede.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini-nosupersede.yaml new file mode 100644 index 00000000..73fc0aff --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini-nosupersede.yaml @@ -0,0 +1,16 @@ +# Knowl, supersession OFF -- the ablation arm. +# +# Every value except the knowl_* pair is copied from the baselines rather than chosen: +# temperature and input_length_limit match every gpt-4o-mini config in the repo, buffer_length +# matches Simple_rag_bm25, and retrieve_num 10 is what BM25, Zep, Cognee, HippoRAG-v2, RAPTOR, +# GraphRAG, Self-RAG and all four embedding baselines use. Mem0 is the outlier at 100. +agent_name: Agentic_memory_knowl_nosupersede +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/knowl-gpt-4o-mini-nosupersede + +retrieve_num: 10 +knowl_supersede: false +knowl_vector: true diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini.yaml new file mode 100644 index 00000000..2741c7d0 --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini.yaml @@ -0,0 +1,16 @@ +# Knowl, supersession ON. This is the published arm. +# +# Every value except the knowl_* pair is copied from the baselines rather than chosen: +# temperature and input_length_limit match every gpt-4o-mini config in the repo, buffer_length +# matches Simple_rag_bm25, and retrieve_num 10 is what BM25, Zep, Cognee, HippoRAG-v2, RAPTOR, +# GraphRAG, Self-RAG and all four embedding baselines use. Mem0 is the outlier at 100. +agent_name: Agentic_memory_knowl +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/knowl-gpt-4o-mini + +retrieve_num: 10 +knowl_supersede: true +knowl_vector: true diff --git a/methods/agentmemory.py b/methods/agentmemory.py new file mode 100644 index 00000000..5e244724 --- /dev/null +++ b/methods/agentmemory.py @@ -0,0 +1,190 @@ +"""agentmemory as a memory method for MemoryAgentBench. + +agentmemory (github.com/rohitg00/agentmemory) is a Node service built on the iii engine. It is +driven here over its REST surface, so nothing is installed into this venv: + + cd /path/to/agentmemory && node dist/cli.mjs # REST on :3111 + export AGENTMEMORY_URL=http://127.0.0.1:3111 # optional, this is the default + +It is not evaluated on MemoryAgentBench upstream -- their published numbers are LongMemEval-S +R@5, a retrieval-recall metric on a different task -- so this is a new measurement rather than a +reproduction, and there is no vendor figure to check it against. + +NORMALIZED INPUT. Both systems receive the identical parsed fact list, in context order, one +record per fact. `parse_fact_lines` below is a faithful port of `facts.ts:parseFactLines` from the +Knowl repository, marker rule included. That is the standing benchmark decision: normalized +retrieval compares identical prepared records with no system-specific extraction, so neither side +gets a cleaner corpus than the other. + +Feeding raw 4096-char chunks instead was considered and rejected. agentmemory stores one memory +per `remember` call, so a chunk would land as a single record holding ~70 facts -- supersession +could never fire and retrieval would return a wall of text. That would measure our chunking +choice, not their memory. + +ISOLATION. Every run writes under a unique `project`. agentmemory's supersession guard skips a +candidate only when both sides carry an explicit and different project (an unscoped record is +treated as a wildcard), so a per-run project name keeps runs from seeing each other as long as +every write is scoped -- which it is here. +""" + +import json +import os +import re +import time +import urllib.error +import urllib.request + +DEFAULT_URL = "http://127.0.0.1:3111" + + +def _post(base_url, path, payload, timeout=120): + data = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + f"{base_url}{path}", + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + + +def strip_trailing_period(text): + text = text.strip() + return text[:-1].strip() if text.endswith(".") else text + + +def parse_fact_lines(context): + """Port of facts.ts:parseFactLines. + + The CR context is a numbered list, `0.` through `N.`. A marker counts only when its number is + the one expected next, so a stray "3." inside a fact's own text is kept as text rather than + splitting it, and any header before "0." is dropped for free. + """ + starts = [] + expected = 0 + for match in re.finditer(r"(\d+)\.", context): + if int(match.group(1)) != expected: + continue + starts.append(match.end()) + expected += 1 + + if not starts: + return [] + + facts = [] + for position, start in enumerate(starts): + if position + 1 < len(starts): + end = context.rfind(f"{position + 1}.", 0, starts[position + 1]) + else: + end = len(context) + facts.append(strip_trailing_period(context[start:end])) + return [f for f in facts if f] + + +class AgentMemoryClient: + def __init__(self, base_url, project): + self.base_url = base_url.rstrip("/") + self.project = project + self.chunks = [] + self.flushed = False + self.facts = 0 + self.superseded = 0 + + def add(self, text): + self.chunks.append(text) + + def flush(self): + """Write every parsed fact, in context order. Idempotent, like the Knowl bridge's flush. + + Order is the only recency signal the task provides -- nothing marks a fact as an update, + which is the whole point of FactConsolidation. + """ + if self.flushed: + return {"facts": self.facts, "superseded": self.superseded} + + facts = parse_fact_lines("".join(self.chunks)) + superseded = 0 + for fact in facts: + result = _post( + self.base_url, + "/agentmemory/remember", + {"content": fact, "project": self.project}, + ) + memory = result.get("memory") or {} + if memory.get("supersedes"): + superseded += 1 + + self.facts = len(facts) + self.superseded = superseded + self.flushed = True + print(f"\nagentmemory flush: {self.facts} facts, {superseded} superseded at write\n") + return {"facts": self.facts, "superseded": superseded} + + def query(self, text, k): + result = _post( + self.base_url, + "/agentmemory/search", + {"query": text, "limit": k, "project": self.project}, + ) + contents = [] + for row in (result.get("results") or [])[:k]: + observation = row.get("observation") or {} + content = observation.get("narrative") or observation.get("title") or "" + if not content: + facts = observation.get("facts") or [] + content = "\n".join(facts) + if content: + contents.append(content) + return contents + + +def initialize_agentmemory_agent(agent, agent_config=None): + config = agent_config or {} + agent.retrieve_num = config["retrieve_num"] + agent.context = "" + agent.agent_start_time = time.time() + + base_url = os.environ.get("AGENTMEMORY_URL", DEFAULT_URL) + project = f"mab_{agent.sub_dataset}_{os.getpid()}_{int(time.time())}" + agent.agentmemory = AgentMemoryClient(base_url, project) + print(f"\n\nagentmemory at {base_url}, project={project}\n\n") + + +def handle_agentmemory_agent(agent, message, memorizing, query_id, context_id): + """Mirror `_handle_bm25_rag`: same query extraction, same reader assembly.""" + from methods.knowl import build_reader_messages, format_retrieval_memory_string + from utils.templates import get_template + + if memorizing: + agent.agentmemory.add(message) + return "Memorized" + + start_time = time.time() + stats = agent.agentmemory.flush() + memory_construction_time = time.time() - start_time + + retrieval_query = agent._extract_retrieval_query(message) + contents = agent.agentmemory.query(retrieval_query, agent.retrieve_num) + retrieval_memory_string = format_retrieval_memory_string(contents) + + system_message = get_template(agent.sub_dataset, "system", agent.agent_name) + format_message = build_reader_messages(retrieval_memory_string, message, system_message) + + response = agent._create_oai_client().chat.completions.create( + model=agent.model, + messages=format_message, + temperature=agent.temperature, + max_tokens=agent.max_tokens if "gpt-4" in agent.model else None, + ) + + query_time_len = time.time() - start_time - memory_construction_time + print(f"\nagentmemory stats: {stats}\n") + + return agent._create_standard_response( + response.choices[0].message.content, + response.usage.prompt_tokens, + response.usage.completion_tokens, + memory_construction_time, + query_time_len, + ) diff --git a/methods/knowl.py b/methods/knowl.py new file mode 100644 index 00000000..8e430477 --- /dev/null +++ b/methods/knowl.py @@ -0,0 +1,213 @@ +"""Knowl as an agentic-memory method for MemoryAgentBench. + +Knowl is a local-first project-memory engine (TypeScript + SQLite). It is driven here through a +small newline-delimited JSON bridge over stdio, built from the Knowl repository: + + npx tsup benchmarks/memoryagentbench/mab-bridge.ts --format esm --outDir .benchmark-dist --no-dts + +Point KNOWL_BRIDGE at the resulting mab-bridge.js. + +What makes Knowl interesting on FactConsolidation is that it resolves conflicts at WRITE time: a +fact whose subject+relation matches one already stored retires the earlier record, so the stale +value is never a retrieval candidate. Set KNOWL_SUPERSEDE=0 for the ablation that switches this +off and leaves both values active -- same corpus, same retrieval, governance toggled. + +Every response line from the bridge is prefixed with a sentinel. The embedding runtime and the +SQLite bindings both write to stdout at will, and an unframed protocol would eventually swallow a +stray log line and desynchronise mid-run -- a failure that surfaces as a plausible score rather +than an error. +""" + +import json +import os +import subprocess + +SENTINEL = "@@KNOWL@@" + + +class KnowlBridge: + """Owns the Node subprocess and speaks the line protocol to it.""" + + def __init__(self, supersede=True, vector=True): + bridge_path = os.environ.get("KNOWL_BRIDGE") + if not bridge_path: + raise RuntimeError( + "Set KNOWL_BRIDGE to the built mab-bridge.js (npx tsup " + "benchmarks/memoryagentbench/mab-bridge.ts --format esm --outDir .benchmark-dist --no-dts)" + ) + # The bridge resolves its embedding profile from the Knowl project it belongs to, via + # findProjectRoot(process.cwd()). Inheriting MemoryAgentBench's cwd makes it look for a + # .knowl next to agent.py and fail. The measured store is a fresh temp dir either way -- + # this only decides which config the embedding preset is read from, and that must be the + # Knowl repo's, so the run is reproducible from a checkout rather than from a local dir. + project_root = os.environ.get("KNOWL_PROJECT") or os.path.dirname( + os.path.dirname(os.path.abspath(bridge_path)) + ) + self.proc = subprocess.Popen( + ["node", bridge_path], + cwd=project_root, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=None, + text=True, + encoding="utf-8", + bufsize=1, + ) + self._call({"op": "init", "supersede": supersede, "vector": vector}) + + def _call(self, payload): + self.proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") + self.proc.stdin.flush() + # Skip anything the runtime printed that is not ours. Only sentinel lines are protocol. + while True: + line = self.proc.stdout.readline() + if not line: + code = self.proc.poll() + if code is not None: + raise RuntimeError(f"Knowl bridge exited with code {code}") + raise RuntimeError("Knowl bridge closed stdout unexpectedly") + line = line.strip() + if not line.startswith(SENTINEL): + continue + result = json.loads(line[len(SENTINEL):].strip()) + if not result.get("ok"): + raise RuntimeError(f"Knowl bridge error: {result.get('error')}") + return result + + def add(self, text): + self._call({"op": "add", "text": text}) + + def flush(self): + """Parse the buffered stream and write every fact. + + Idempotent: the construction-time stamp and the first query both reach for it. + + Ingestion is deferred until the whole stream is in hand: the titling rule derives each + fact's subject+relation by shared-prefix discovery across the WHOLE fact list, so it + cannot run chunk by chunk. Calling this explicitly at the end of memorisation is what + keeps MemoryAgentBench's `memory_construction_time` honest -- without it the harness + reports ~0.01s and buries the real ingest cost inside the latency of question 1. + """ + return self._call({"op": "flush"}) + + def query(self, text, k): + return self._call({"op": "query", "text": text, "k": k})["contents"] + + def close(self): + try: + self._call({"op": "close"}) + except Exception: + pass + try: + self.proc.stdin.close() + self.proc.wait(timeout=30) + except Exception: + self.proc.kill() + + +def build_reader_messages(retrieval_memory_string, message, system_message): + """Assemble the reader prompt. + + Default layout is byte-identical to MemoryAgentBench's own RAG handler + (`ask_llm_message = retrieval_memory_string + "\\n" + message`), so a Knowl number sits + beside the published baselines on equal terms. The task instruction therefore TRAILS the + retrieved facts, and the system message is only the generic "you are a helpful assistant". + + KNOWL_MAB_READER_LAYOUT=system-first is a diagnostic, not a competing result: it moves the + same instruction text into a system message ahead of the facts. It is our construction, not + a standard, and any number produced under it must be labelled as such. + """ + if os.environ.get("KNOWL_MAB_READER_LAYOUT") == "system-first": + return [ + {"role": "system", "content": message}, + {"role": "user", "content": retrieval_memory_string}, + ] + ask_llm_message = retrieval_memory_string + "\n" + message + return [ + {"role": "system", "content": system_message}, + {"role": "user", "content": ask_llm_message}, + ] + + +def format_retrieval_memory_string(contents): + """Match `_handle_bm25_rag` exactly: each item gets a trailing newline, then 'Memory i:' labels.""" + retrieval_context = [f"{text}\n" for text in contents] + return "\n".join(f"Memory {i + 1}:\n{text}" for i, text in enumerate(retrieval_context)) + + +# ── the two entry points agent.py delegates to ───────────────────────────────── +# +# The logic lives here rather than in agent.py so the patch applied to MemoryAgentBench stays +# three lines. Upstream moves -- this clone already took a renumbered table and a new baseline -- +# and a small patch survives a rebase where a large one does not. + +def initialize_knowl_agent(agent, agent_config=None): + """Start the bridge. + + The arm is chosen by `knowl_supersede` in the agent config -- that is what distinguishes the + two checked-in YAMLs, so a run is reproducible from the config alone. KNOWL_SUPERSEDE=0 still + overrides, for a one-off ablation without editing a file. + """ + config = agent_config or {} + # Set the same fields every other memory agent's initialiser sets (see _initialize_mem0_agent): + # the memory-agent path does not run _initialize_rag_agent, so nothing else assigns these. + agent.retrieve_num = config["retrieve_num"] + agent.context = "" + agent.agent_start_time = __import__("time").time() + + supersede = bool(config.get("knowl_supersede", True)) + vector = bool(config.get("knowl_vector", True)) + if os.environ.get("KNOWL_SUPERSEDE") == "0": + supersede = False + agent.knowl = KnowlBridge(supersede=supersede, vector=vector) + print("Knowl bridge up, supersede=%s vector=%s" % (supersede, vector)) + + +def handle_knowl_agent(agent, message, memorizing, query_id, context_id): + """Mirror `_handle_bm25_rag`: same retrieval-query extraction, same reader assembly. + + The only deliberate difference from the RAG handlers is that ingestion is buffered and + flushed once, which is a property of the system under test rather than of the harness. + """ + import time + + if memorizing: + agent.knowl.add(message) + return "Memorized" + + start_time = time.time() + + # Flush here, not lazily inside the first query, so the cost lands in + # memory_construction_time where the harness reports it. + stats = agent.knowl.flush() + memory_construction_time = time.time() - start_time + + # Identical to every RAG baseline: MAB wraps each question in ~200 tokens of task + # boilerplate that is byte-identical across questions, so retrieving on the raw message is + # retrieving on noise. See agent.py `_extract_retrieval_query`. + retrieval_query = agent._extract_retrieval_query(message) + + contents = agent.knowl.query(retrieval_query, agent.retrieve_num) + retrieval_memory_string = format_retrieval_memory_string(contents) + + from utils.templates import get_template + system_message = get_template(agent.sub_dataset, "system", agent.agent_name) + format_message = build_reader_messages(retrieval_memory_string, message, system_message) + + response = agent._create_oai_client().chat.completions.create( + model=agent.model, + messages=format_message, + temperature=agent.temperature, + max_tokens=agent.max_tokens if "gpt-4" in agent.model else None, + ) + + query_time_len = time.time() - start_time - memory_construction_time + print(f"\nknowl stats: {stats}\n") + + return agent._create_standard_response( + response.choices[0].message.content, + response.usage.prompt_tokens, + response.usage.completion_tokens, + memory_construction_time, + query_time_len, + )