diff --git a/agent.py b/agent.py index d3cef2dd..ae859fb5 100644 --- a/agent.py +++ b/agent.py @@ -74,6 +74,12 @@ def _initialize_agent_by_type(self, agent_config, dataset_config): self._initialize_cognee_agent(agent_config, dataset_config) elif self._is_agent_type("zep"): self._initialize_zep_agent(agent_config) + elif self._is_agent_type("knowl"): + from methods.knowl import initialize_knowl_agent + initialize_knowl_agent(self, agent_config) + elif self._is_agent_type("agentmemory"): + from methods.agentmemory import initialize_agentmemory_agent + initialize_agentmemory_agent(self, agent_config) elif self._is_agent_type("rag"): self._initialize_rag_agent(agent_config, dataset_config) else: @@ -271,7 +277,7 @@ def send_message(self, message, memorizing=False, query_id=None, context_id=None # Route to appropriate agent handler based on agent type if 'Long_context_agent' in self.agent_name: return self._handle_long_context_agent(message, memorizing) - elif any(self._is_agent_type(agent_type) for agent_type in ["letta", "cognee", "mem0", "zep"]): + elif any(self._is_agent_type(agent_type) for agent_type in ["letta", "cognee", "mem0", "zep", "knowl", "agentmemory"]): return self._handle_memory_agent(message, memorizing, query_id, context_id) elif self._is_agent_type("rag"): return self._handle_rag_agent(message, memorizing, query_id, context_id) @@ -406,6 +412,12 @@ def _handle_memory_agent(self, message, memorizing, query_id, context_id): return self._handle_mem0_agent(message, memorizing, query_id, context_id) elif self._is_agent_type("zep"): return self._handle_zep_agent(message, memorizing, query_id, context_id) + elif self._is_agent_type("knowl"): + from methods.knowl import handle_knowl_agent + return handle_knowl_agent(self, message, memorizing, query_id, context_id) + elif self._is_agent_type("agentmemory"): + from methods.agentmemory import handle_agentmemory_agent + return handle_agentmemory_agent(self, message, memorizing, query_id, context_id) else: raise NotImplementedError(f"Memory agent type not supported: {self.agent_name}") diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml new file mode 100644 index 00000000..ca15c9a1 --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/AgentMemory_gpt-4o-mini.yaml @@ -0,0 +1,11 @@ +# agentmemory v0.9.29, driven over REST. Every value except the output_dir is copied from +# Simple_rag_bm25 so the row sits beside the published baselines on the same terms: +# retrieve_num 10 is what BM25, Zep, Cognee, HippoRAG-v2 and the embedding baselines use. +agent_name: Agentic_memory_agentmemory +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/agentmemory-gpt-4o-mini + +retrieve_num: 10 diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini-nosupersede.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini-nosupersede.yaml new file mode 100644 index 00000000..73fc0aff --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini-nosupersede.yaml @@ -0,0 +1,16 @@ +# Knowl, supersession OFF -- the ablation arm. +# +# Every value except the knowl_* pair is copied from the baselines rather than chosen: +# temperature and input_length_limit match every gpt-4o-mini config in the repo, buffer_length +# matches Simple_rag_bm25, and retrieve_num 10 is what BM25, Zep, Cognee, HippoRAG-v2, RAPTOR, +# GraphRAG, Self-RAG and all four embedding baselines use. Mem0 is the outlier at 100. +agent_name: Agentic_memory_knowl_nosupersede +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/knowl-gpt-4o-mini-nosupersede + +retrieve_num: 10 +knowl_supersede: false +knowl_vector: true diff --git a/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini.yaml b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini.yaml new file mode 100644 index 00000000..2741c7d0 --- /dev/null +++ b/configs/agent_conf/RAG_Agents/gpt-4o-mini/Knowl_gpt-4o-mini.yaml @@ -0,0 +1,16 @@ +# Knowl, supersession ON. This is the published arm. +# +# Every value except the knowl_* pair is copied from the baselines rather than chosen: +# temperature and input_length_limit match every gpt-4o-mini config in the repo, buffer_length +# matches Simple_rag_bm25, and retrieve_num 10 is what BM25, Zep, Cognee, HippoRAG-v2, RAPTOR, +# GraphRAG, Self-RAG and all four embedding baselines use. Mem0 is the outlier at 100. +agent_name: Agentic_memory_knowl +model: gpt-4o-mini +temperature: 0.7 +input_length_limit: 10000000 +buffer_length: 200 +output_dir: ./outputs/knowl-gpt-4o-mini + +retrieve_num: 10 +knowl_supersede: true +knowl_vector: true diff --git a/methods/agentmemory.py b/methods/agentmemory.py new file mode 100644 index 00000000..5e244724 --- /dev/null +++ b/methods/agentmemory.py @@ -0,0 +1,190 @@ +"""agentmemory as a memory method for MemoryAgentBench. + +agentmemory (github.com/rohitg00/agentmemory) is a Node service built on the iii engine. It is +driven here over its REST surface, so nothing is installed into this venv: + + cd /path/to/agentmemory && node dist/cli.mjs # REST on :3111 + export AGENTMEMORY_URL=http://127.0.0.1:3111 # optional, this is the default + +It is not evaluated on MemoryAgentBench upstream -- their published numbers are LongMemEval-S +R@5, a retrieval-recall metric on a different task -- so this is a new measurement rather than a +reproduction, and there is no vendor figure to check it against. + +NORMALIZED INPUT. Both systems receive the identical parsed fact list, in context order, one +record per fact. `parse_fact_lines` below is a faithful port of `facts.ts:parseFactLines` from the +Knowl repository, marker rule included. That is the standing benchmark decision: normalized +retrieval compares identical prepared records with no system-specific extraction, so neither side +gets a cleaner corpus than the other. + +Feeding raw 4096-char chunks instead was considered and rejected. agentmemory stores one memory +per `remember` call, so a chunk would land as a single record holding ~70 facts -- supersession +could never fire and retrieval would return a wall of text. That would measure our chunking +choice, not their memory. + +ISOLATION. Every run writes under a unique `project`. agentmemory's supersession guard skips a +candidate only when both sides carry an explicit and different project (an unscoped record is +treated as a wildcard), so a per-run project name keeps runs from seeing each other as long as +every write is scoped -- which it is here. +""" + +import json +import os +import re +import time +import urllib.error +import urllib.request + +DEFAULT_URL = "http://127.0.0.1:3111" + + +def _post(base_url, path, payload, timeout=120): + data = json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + f"{base_url}{path}", + data=data, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + + +def strip_trailing_period(text): + text = text.strip() + return text[:-1].strip() if text.endswith(".") else text + + +def parse_fact_lines(context): + """Port of facts.ts:parseFactLines. + + The CR context is a numbered list, `0.` through `N.`. A marker counts only when its number is + the one expected next, so a stray "3." inside a fact's own text is kept as text rather than + splitting it, and any header before "0." is dropped for free. + """ + starts = [] + expected = 0 + for match in re.finditer(r"(\d+)\.", context): + if int(match.group(1)) != expected: + continue + starts.append(match.end()) + expected += 1 + + if not starts: + return [] + + facts = [] + for position, start in enumerate(starts): + if position + 1 < len(starts): + end = context.rfind(f"{position + 1}.", 0, starts[position + 1]) + else: + end = len(context) + facts.append(strip_trailing_period(context[start:end])) + return [f for f in facts if f] + + +class AgentMemoryClient: + def __init__(self, base_url, project): + self.base_url = base_url.rstrip("/") + self.project = project + self.chunks = [] + self.flushed = False + self.facts = 0 + self.superseded = 0 + + def add(self, text): + self.chunks.append(text) + + def flush(self): + """Write every parsed fact, in context order. Idempotent, like the Knowl bridge's flush. + + Order is the only recency signal the task provides -- nothing marks a fact as an update, + which is the whole point of FactConsolidation. + """ + if self.flushed: + return {"facts": self.facts, "superseded": self.superseded} + + facts = parse_fact_lines("".join(self.chunks)) + superseded = 0 + for fact in facts: + result = _post( + self.base_url, + "/agentmemory/remember", + {"content": fact, "project": self.project}, + ) + memory = result.get("memory") or {} + if memory.get("supersedes"): + superseded += 1 + + self.facts = len(facts) + self.superseded = superseded + self.flushed = True + print(f"\nagentmemory flush: {self.facts} facts, {superseded} superseded at write\n") + return {"facts": self.facts, "superseded": superseded} + + def query(self, text, k): + result = _post( + self.base_url, + "/agentmemory/search", + {"query": text, "limit": k, "project": self.project}, + ) + contents = [] + for row in (result.get("results") or [])[:k]: + observation = row.get("observation") or {} + content = observation.get("narrative") or observation.get("title") or "" + if not content: + facts = observation.get("facts") or [] + content = "\n".join(facts) + if content: + contents.append(content) + return contents + + +def initialize_agentmemory_agent(agent, agent_config=None): + config = agent_config or {} + agent.retrieve_num = config["retrieve_num"] + agent.context = "" + agent.agent_start_time = time.time() + + base_url = os.environ.get("AGENTMEMORY_URL", DEFAULT_URL) + project = f"mab_{agent.sub_dataset}_{os.getpid()}_{int(time.time())}" + agent.agentmemory = AgentMemoryClient(base_url, project) + print(f"\n\nagentmemory at {base_url}, project={project}\n\n") + + +def handle_agentmemory_agent(agent, message, memorizing, query_id, context_id): + """Mirror `_handle_bm25_rag`: same query extraction, same reader assembly.""" + from methods.knowl import build_reader_messages, format_retrieval_memory_string + from utils.templates import get_template + + if memorizing: + agent.agentmemory.add(message) + return "Memorized" + + start_time = time.time() + stats = agent.agentmemory.flush() + memory_construction_time = time.time() - start_time + + retrieval_query = agent._extract_retrieval_query(message) + contents = agent.agentmemory.query(retrieval_query, agent.retrieve_num) + retrieval_memory_string = format_retrieval_memory_string(contents) + + system_message = get_template(agent.sub_dataset, "system", agent.agent_name) + format_message = build_reader_messages(retrieval_memory_string, message, system_message) + + response = agent._create_oai_client().chat.completions.create( + model=agent.model, + messages=format_message, + temperature=agent.temperature, + max_tokens=agent.max_tokens if "gpt-4" in agent.model else None, + ) + + query_time_len = time.time() - start_time - memory_construction_time + print(f"\nagentmemory stats: {stats}\n") + + return agent._create_standard_response( + response.choices[0].message.content, + response.usage.prompt_tokens, + response.usage.completion_tokens, + memory_construction_time, + query_time_len, + ) diff --git a/methods/knowl.py b/methods/knowl.py new file mode 100644 index 00000000..8e430477 --- /dev/null +++ b/methods/knowl.py @@ -0,0 +1,213 @@ +"""Knowl as an agentic-memory method for MemoryAgentBench. + +Knowl is a local-first project-memory engine (TypeScript + SQLite). It is driven here through a +small newline-delimited JSON bridge over stdio, built from the Knowl repository: + + npx tsup benchmarks/memoryagentbench/mab-bridge.ts --format esm --outDir .benchmark-dist --no-dts + +Point KNOWL_BRIDGE at the resulting mab-bridge.js. + +What makes Knowl interesting on FactConsolidation is that it resolves conflicts at WRITE time: a +fact whose subject+relation matches one already stored retires the earlier record, so the stale +value is never a retrieval candidate. Set KNOWL_SUPERSEDE=0 for the ablation that switches this +off and leaves both values active -- same corpus, same retrieval, governance toggled. + +Every response line from the bridge is prefixed with a sentinel. The embedding runtime and the +SQLite bindings both write to stdout at will, and an unframed protocol would eventually swallow a +stray log line and desynchronise mid-run -- a failure that surfaces as a plausible score rather +than an error. +""" + +import json +import os +import subprocess + +SENTINEL = "@@KNOWL@@" + + +class KnowlBridge: + """Owns the Node subprocess and speaks the line protocol to it.""" + + def __init__(self, supersede=True, vector=True): + bridge_path = os.environ.get("KNOWL_BRIDGE") + if not bridge_path: + raise RuntimeError( + "Set KNOWL_BRIDGE to the built mab-bridge.js (npx tsup " + "benchmarks/memoryagentbench/mab-bridge.ts --format esm --outDir .benchmark-dist --no-dts)" + ) + # The bridge resolves its embedding profile from the Knowl project it belongs to, via + # findProjectRoot(process.cwd()). Inheriting MemoryAgentBench's cwd makes it look for a + # .knowl next to agent.py and fail. The measured store is a fresh temp dir either way -- + # this only decides which config the embedding preset is read from, and that must be the + # Knowl repo's, so the run is reproducible from a checkout rather than from a local dir. + project_root = os.environ.get("KNOWL_PROJECT") or os.path.dirname( + os.path.dirname(os.path.abspath(bridge_path)) + ) + self.proc = subprocess.Popen( + ["node", bridge_path], + cwd=project_root, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=None, + text=True, + encoding="utf-8", + bufsize=1, + ) + self._call({"op": "init", "supersede": supersede, "vector": vector}) + + def _call(self, payload): + self.proc.stdin.write(json.dumps(payload, ensure_ascii=False) + "\n") + self.proc.stdin.flush() + # Skip anything the runtime printed that is not ours. Only sentinel lines are protocol. + while True: + line = self.proc.stdout.readline() + if not line: + code = self.proc.poll() + if code is not None: + raise RuntimeError(f"Knowl bridge exited with code {code}") + raise RuntimeError("Knowl bridge closed stdout unexpectedly") + line = line.strip() + if not line.startswith(SENTINEL): + continue + result = json.loads(line[len(SENTINEL):].strip()) + if not result.get("ok"): + raise RuntimeError(f"Knowl bridge error: {result.get('error')}") + return result + + def add(self, text): + self._call({"op": "add", "text": text}) + + def flush(self): + """Parse the buffered stream and write every fact. + + Idempotent: the construction-time stamp and the first query both reach for it. + + Ingestion is deferred until the whole stream is in hand: the titling rule derives each + fact's subject+relation by shared-prefix discovery across the WHOLE fact list, so it + cannot run chunk by chunk. Calling this explicitly at the end of memorisation is what + keeps MemoryAgentBench's `memory_construction_time` honest -- without it the harness + reports ~0.01s and buries the real ingest cost inside the latency of question 1. + """ + return self._call({"op": "flush"}) + + def query(self, text, k): + return self._call({"op": "query", "text": text, "k": k})["contents"] + + def close(self): + try: + self._call({"op": "close"}) + except Exception: + pass + try: + self.proc.stdin.close() + self.proc.wait(timeout=30) + except Exception: + self.proc.kill() + + +def build_reader_messages(retrieval_memory_string, message, system_message): + """Assemble the reader prompt. + + Default layout is byte-identical to MemoryAgentBench's own RAG handler + (`ask_llm_message = retrieval_memory_string + "\\n" + message`), so a Knowl number sits + beside the published baselines on equal terms. The task instruction therefore TRAILS the + retrieved facts, and the system message is only the generic "you are a helpful assistant". + + KNOWL_MAB_READER_LAYOUT=system-first is a diagnostic, not a competing result: it moves the + same instruction text into a system message ahead of the facts. It is our construction, not + a standard, and any number produced under it must be labelled as such. + """ + if os.environ.get("KNOWL_MAB_READER_LAYOUT") == "system-first": + return [ + {"role": "system", "content": message}, + {"role": "user", "content": retrieval_memory_string}, + ] + ask_llm_message = retrieval_memory_string + "\n" + message + return [ + {"role": "system", "content": system_message}, + {"role": "user", "content": ask_llm_message}, + ] + + +def format_retrieval_memory_string(contents): + """Match `_handle_bm25_rag` exactly: each item gets a trailing newline, then 'Memory i:' labels.""" + retrieval_context = [f"{text}\n" for text in contents] + return "\n".join(f"Memory {i + 1}:\n{text}" for i, text in enumerate(retrieval_context)) + + +# ── the two entry points agent.py delegates to ───────────────────────────────── +# +# The logic lives here rather than in agent.py so the patch applied to MemoryAgentBench stays +# three lines. Upstream moves -- this clone already took a renumbered table and a new baseline -- +# and a small patch survives a rebase where a large one does not. + +def initialize_knowl_agent(agent, agent_config=None): + """Start the bridge. + + The arm is chosen by `knowl_supersede` in the agent config -- that is what distinguishes the + two checked-in YAMLs, so a run is reproducible from the config alone. KNOWL_SUPERSEDE=0 still + overrides, for a one-off ablation without editing a file. + """ + config = agent_config or {} + # Set the same fields every other memory agent's initialiser sets (see _initialize_mem0_agent): + # the memory-agent path does not run _initialize_rag_agent, so nothing else assigns these. + agent.retrieve_num = config["retrieve_num"] + agent.context = "" + agent.agent_start_time = __import__("time").time() + + supersede = bool(config.get("knowl_supersede", True)) + vector = bool(config.get("knowl_vector", True)) + if os.environ.get("KNOWL_SUPERSEDE") == "0": + supersede = False + agent.knowl = KnowlBridge(supersede=supersede, vector=vector) + print("Knowl bridge up, supersede=%s vector=%s" % (supersede, vector)) + + +def handle_knowl_agent(agent, message, memorizing, query_id, context_id): + """Mirror `_handle_bm25_rag`: same retrieval-query extraction, same reader assembly. + + The only deliberate difference from the RAG handlers is that ingestion is buffered and + flushed once, which is a property of the system under test rather than of the harness. + """ + import time + + if memorizing: + agent.knowl.add(message) + return "Memorized" + + start_time = time.time() + + # Flush here, not lazily inside the first query, so the cost lands in + # memory_construction_time where the harness reports it. + stats = agent.knowl.flush() + memory_construction_time = time.time() - start_time + + # Identical to every RAG baseline: MAB wraps each question in ~200 tokens of task + # boilerplate that is byte-identical across questions, so retrieving on the raw message is + # retrieving on noise. See agent.py `_extract_retrieval_query`. + retrieval_query = agent._extract_retrieval_query(message) + + contents = agent.knowl.query(retrieval_query, agent.retrieve_num) + retrieval_memory_string = format_retrieval_memory_string(contents) + + from utils.templates import get_template + system_message = get_template(agent.sub_dataset, "system", agent.agent_name) + format_message = build_reader_messages(retrieval_memory_string, message, system_message) + + response = agent._create_oai_client().chat.completions.create( + model=agent.model, + messages=format_message, + temperature=agent.temperature, + max_tokens=agent.max_tokens if "gpt-4" in agent.model else None, + ) + + query_time_len = time.time() - start_time - memory_construction_time + print(f"\nknowl stats: {stats}\n") + + return agent._create_standard_response( + response.choices[0].message.content, + response.usage.prompt_tokens, + response.usage.completion_tokens, + memory_construction_time, + query_time_len, + )