From 1290859509819f82c739ba5480b0b0eb1aad5b6d Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Mon, 17 Aug 2026 19:58:02 +0300 Subject: [PATCH 01/34] feat(memory): centralize persistent memory access --- lib_omegaclaw.metta | 1 + src/helper.py | 4 +- src/memory.metta | 12 ++---- src/memory_gateway.py | 80 ++++++++++++++++++++++++++++++++++++ src/memory_layout.py | 19 +++++++++ tests/test_memory_gateway.py | 42 +++++++++++++++++++ tests/test_memory_layout.py | 19 +++++++++ 7 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 src/memory_gateway.py create mode 100644 src/memory_layout.py create mode 100644 tests/test_memory_gateway.py create mode 100644 tests/test_memory_layout.py diff --git a/lib_omegaclaw.metta b/lib_omegaclaw.metta index b8cc0a44..e0dc2884 100644 --- a/lib_omegaclaw.metta +++ b/lib_omegaclaw.metta @@ -22,6 +22,7 @@ !(import! &self (library OmegaClaw-Core ./src/fileio.py)) !(import! &self (library OmegaClaw-Core ./src/skills)) !(import! &self (library OmegaClaw-Core ./src/websearch.py)) +!(import! &self (library OmegaClaw-Core ./src/memory_gateway.py)) !(import! &self (library OmegaClaw-Core ./src/memory)) !(import! &self (library OmegaClaw-Core ./src/context)) !(import! &self (library OmegaClaw-Core ./src/loop)) diff --git a/src/helper.py b/src/helper.py index 6056568f..1f90a1a8 100644 --- a/src/helper.py +++ b/src/helper.py @@ -9,8 +9,10 @@ try: from src.logger import get_logger + from src.memory_layout import history_path except ModuleNotFoundError: # running this file directly as a script from logger import get_logger + from memory_layout import history_path logger = get_logger(__name__) @@ -63,7 +65,7 @@ def extract_timestamp(line): def around_time(needle_time_str, k): needle_time_str = needle_time_str.replace(r'\"', '').replace('"', '').strip() - filename = "repos/OmegaClaw-Core/memory/history.metta" + filename = history_path() target = datetime.strptime(needle_time_str, "%Y-%m-%d %H:%M:%S") best_lineno = None best_line = None diff --git a/src/memory.metta b/src/memory.metta index 39795f29..ecb95a28 100644 --- a/src/memory.metta +++ b/src/memory.metta @@ -27,11 +27,7 @@ ""))))) (= (getHistory) - (let $history_file - (library OmegaClaw-Core ./memory/history.metta) - (if (exists-file $history_file) ;Safely read the history. Return an empty string if the file does not exist. - (read_file_tail $history_file (maxHistory)) - ""))) + (py-call (memory_gateway.history_tail (maxHistory)))) (= (addToHistory $lastmessage $response $sexpr $msgnew) (if $msgnew @@ -48,14 +44,14 @@ (py-call (rag.openai_embed (string-safe $str))))) (= (appendToHistory $addition) - (append-file-raw (library OmegaClaw-Core ./memory/history.metta) (swrite $addition))) + (py-call (memory_gateway.append_history (swrite $addition)))) (= (remember $str) - (progn (py-call (lib_chromadb.remember $str (embed $str) (get_time_as_string))) + (progn (py-call (memory_gateway.remember $str (embed $str) (get_time_as_string))) REMEMBER-SUCCESS)) (= (query $str) - (py-call (lib_chromadb.query (embed $str) (maxRecallItems)))) + (py-call (memory_gateway.query (embed $str) (maxRecallItems)))) (= (episodes $time) (py-call (helper.around_time $time (maxEpisodeRecallLines)))) diff --git a/src/memory_gateway.py b/src/memory_gateway.py new file mode 100644 index 00000000..fff0a8c4 --- /dev/null +++ b/src/memory_gateway.py @@ -0,0 +1,80 @@ +"""Thread-safe access to persistent history and vector memory.""" + +import threading +import uuid +import chromadb + +from src.memory_layout import history_path, chroma_db_path +from src.logger import get_logger + +logger = get_logger(__name__) + +_write_lock = threading.Lock() + +_RECORD_KIND = "user_memory" + +_client: chromadb.ClientAPI | None = None +_collection = None + +def _get_collection(): + """Lazy-initialise the ChromaDB client and collection.""" + global _client, _collection + if _collection is None: + db_path = str(chroma_db_path()) + logger.info(f"memory_gateway: opening ChromaDB at {db_path}") + _client = chromadb.PersistentClient(path=db_path) + _collection = _client.get_or_create_collection( + name="memories", + embedding_function=None, + ) + return _collection + +def append_history(text: str) -> None: + """Append a history entry and its trailing newline, creating the file.""" + path = history_path() + path.parent.mkdir(parents=True, exist_ok=True) + with _write_lock: + with path.open("a", encoding="utf-8") as f: + f.write(text) + f.write("\n") + +def history_tail(max_chars: int) -> str: + """Return the trailing character window of history, or an empty string.""" + path = history_path() + if not path.exists(): + return "" + with path.open("r", encoding="utf-8", errors="replace") as f: + return f.read()[-max_chars:] + +def remember(content: str, embedding: list[float], time: str) -> str: + """Store a user memory record and return its ID.""" + item_id = str(uuid.uuid4()) + with _write_lock: + _get_collection().add( + ids=[item_id], + documents=[content], + embeddings=[embedding], + metadatas=[{"time": time, "record_kind": _RECORD_KIND}], + ) + logger.debug(f"memory_gateway: remembered record {item_id}") + return item_id + +def query(query_embedding: list[float], k: int) -> list[list]: + """Return the k most similar records as [time, content].""" + results = query_with_ids(query_embedding, k) + return [[time, content] for _, time, content in results] + +def query_with_ids(query_embedding: list[float], k: int) -> list[list]: + """Return the k most similar records as [id, time, content].""" + res = _get_collection().query( + query_embeddings=[query_embedding], + n_results=k, + include=["documents", "metadatas", "distances"], + ) + ids = res["ids"][0] + docs = res.get("documents", [[]])[0] + metas = res.get("metadatas", [[]])[0] + return [ + [ids[i], metas[i].get("time") if metas[i] else None, docs[i]] + for i in range(len(ids)) + ] diff --git a/src/memory_layout.py b/src/memory_layout.py new file mode 100644 index 00000000..28a4819b --- /dev/null +++ b/src/memory_layout.py @@ -0,0 +1,19 @@ +import os +import pathlib + +_REPO_ROOT = pathlib.Path(__file__).parent.parent.resolve() + +def memory_dir_path() -> pathlib.Path: + """Return MEMORY_DIR or the repository's memory directory.""" + memory_dir = os.environ.get("MEMORY_DIR") + if memory_dir: + return pathlib.Path(memory_dir).resolve() + return _REPO_ROOT / "memory" + +def history_path() -> pathlib.Path: + """Return the history file path.""" + return memory_dir_path() / "history.metta" + +def chroma_db_path() -> pathlib.Path: + """Return the ChromaDB directory path.""" + return memory_dir_path() / "chroma_db" diff --git a/tests/test_memory_gateway.py b/tests/test_memory_gateway.py new file mode 100644 index 00000000..3b601dcf --- /dev/null +++ b/tests/test_memory_gateway.py @@ -0,0 +1,42 @@ +import importlib +import threading + +import pytest + +EMBEDDING = [0.1, 0.2, 0.3] + + +@pytest.fixture(autouse=True) +def isolated_gateway(tmp_path, monkeypatch): + monkeypatch.setenv("MEMORY_DIR", str(tmp_path)) + import src.memory_layout as layout + importlib.reload(layout) + import src.memory_gateway as gateway + importlib.reload(gateway) + return gateway + + +def test_history_append_and_ltm_query(isolated_gateway, tmp_path): + isolated_gateway.append_history("first") + isolated_gateway.append_history("second") + isolated_gateway.remember("portable fact", EMBEDDING, "2026-01-01") + + assert (tmp_path / "history.metta").read_text() == "first\nsecond\n" + assert isolated_gateway.query(EMBEDDING, 1) == [["2026-01-01", "portable fact"]] + metadata = isolated_gateway._get_collection().get(include=["metadatas"])["metadatas"] + assert metadata[0]["record_kind"] == "user_memory" + + +def test_export_lock_blocks_history_write(isolated_gateway, tmp_path): + finished = threading.Event() + thread = threading.Thread( + target=lambda: (isolated_gateway.append_history("concurrent"), finished.set()) + ) + + with isolated_gateway._write_lock: + thread.start() + assert not finished.wait(0.05) + thread.join(timeout=2) + + assert finished.is_set() + assert (tmp_path / "history.metta").read_text() == "concurrent\n" diff --git a/tests/test_memory_layout.py b/tests/test_memory_layout.py new file mode 100644 index 00000000..fd64d676 --- /dev/null +++ b/tests/test_memory_layout.py @@ -0,0 +1,19 @@ +import importlib + +import src.memory_layout as memory_layout + + +def test_memory_dir_controls_both_persistent_paths(tmp_path, monkeypatch): + monkeypatch.setenv("MEMORY_DIR", str(tmp_path)) + layout = importlib.reload(memory_layout) + + assert layout.history_path() == tmp_path.resolve() / "history.metta" + assert layout.chroma_db_path() == tmp_path.resolve() / "chroma_db" + + +def test_default_paths_are_absolute(monkeypatch): + monkeypatch.delenv("MEMORY_DIR", raising=False) + layout = importlib.reload(memory_layout) + + assert layout.history_path().is_absolute() + assert layout.chroma_db_path().is_absolute() From b034451331bcc303cc63098eb3bb18c64c0b909e Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Mon, 17 Aug 2026 19:58:18 +0300 Subject: [PATCH 02/34] feat(memory): add portable archive import and export --- config/config.yaml | 2 + pyproject.toml | 1 + src/memory_transfer.py | 878 ++++++++++++++++++++++++++++++++++ tests/test_memory_transfer.py | 273 +++++++++++ 4 files changed, 1154 insertions(+) create mode 100644 src/memory_transfer.py create mode 100644 tests/test_memory_transfer.py diff --git a/config/config.yaml b/config/config.yaml index 1034f7a4..d0986a33 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -36,6 +36,8 @@ maxEpisodeRecallLines: 20 maxHistory: 30000 # `Local` (Python-side model) or `OpenAI` (requires `OPENAI_API_KEY`) embeddingprovider: Local +# Enable authenticated /memory-export commands (disabled by default). +memoryExportEnabled: false # Policy diff --git a/pyproject.toml b/pyproject.toml index 748eaf71..2cfa940f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,5 @@ [tool.pytest.ini_options] pythonpath = [ + ".", "src" ] diff --git a/src/memory_transfer.py b/src/memory_transfer.py new file mode 100644 index 00000000..27556ead --- /dev/null +++ b/src/memory_transfer.py @@ -0,0 +1,878 @@ +"""Export and import persistent user memory.""" + +import argparse +import hashlib +import json +import os +import shutil +import tarfile +import tempfile +import threading +import uuid +from collections.abc import Callable +from datetime import datetime, timezone +from pathlib import Path + +import chromadb + +import src.memory_gateway as gateway +from src.memory_layout import chroma_db_path, history_path, memory_dir_path +from src.logger import get_logger +from src.config import config_get_by_key + +logger = get_logger(__name__) + +TRANSFER_DIR = Path("/memory-transfer") +ARCHIVE_FORMAT_VERSION = 1 + +_ALLOWLIST = frozenset([ + "manifest.json", + "history/history.metta", + "vector/collections.json", + "vector/records.jsonl", +]) + +_COMPONENT_FILES = { + "history": {"history/history.metta"}, + "ltm": {"vector/collections.json", "vector/records.jsonl"}, +} + +_MAX_COMPRESSED_BYTES = 500 * 1024 * 1024 # 500 MB +_MAX_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024 # 2 GB + +_TX_MARKER_NAME = ".import_in_progress" +_RECEIPT_DIR_NAME = ".memory_import_receipts" +_ROLLBACK_STATE_NAME = "state.json" + +_REEMBED_BATCH = 64 +_VECTOR_BATCH = 500 + +def is_export_enabled() -> bool: + """Return whether an administrator enabled memory export.""" + env_val = os.environ.get("OMEGACLAW_memoryExportEnabled") + if env_val is not None: + return env_val.strip().lower() == "true" + + value = config_get_by_key("memoryExportEnabled", False) + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() == "true" + return value is True + +def _sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(65536), b""): + h.update(chunk) + return h.hexdigest() + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + +def _archive_name() -> str: + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") + return f"omegaclaw-memory-{timestamp}.tar.gz" + +def _embedding_profile() -> dict: + """Return the active embedding profile used to assess archive compatibility.""" + provider = os.environ.get("EMBEDDING_PROVIDER", "Local") + default_model = "text-embedding-3-large" if provider == "OpenAI" else "intfloat/e5-large-v2" + return { + "provider": provider, + "model": os.environ.get("SENTENCE_TRANSFORMERS_MODEL", default_model), + } + +def _reembed_records(records: list[dict], manifest: dict, + target_dimension: int | None) -> None: + """Regenerate embeddings in bounded batches when archive and runtime differ.""" + source = manifest.get("embedding_info", {}) + active = _embedding_profile() + source_dimension = source.get("vector_dimension") + embeddings_present = all( + record.get("embedding") and len(record["embedding"]) == source_dimension + for record in records + ) + if (source.get("provider") == active["provider"] and + source.get("model") == active["model"] and embeddings_present and + target_dimension in (None, source_dimension)): + return + + logger.warning("memory_transfer: embedding profile mismatch — re-embedding in batches") + from src.rag import local_embed_batch, openai_embed_batch + + embed_fn = openai_embed_batch if active["provider"] == "OpenAI" else local_embed_batch + for i in range(0, len(records), _REEMBED_BATCH): + batch = records[i : i + _REEMBED_BATCH] + embeddings = embed_fn([record["document"] for record in batch]) + for record, embedding in zip(batch, embeddings): + record["embedding"] = embedding + +def _normalise_metadata(metadata: object) -> dict: + """Convert archive metadata into ChromaDB's scalar-only metadata format.""" + if not isinstance(metadata, dict): + raise ValueError("Archive record metadata must be an object") + + normalised = {} + for key, value in metadata.items(): + if not isinstance(key, str) or not key: + raise ValueError("Archive metadata keys must be non-empty strings") + if isinstance(value, (str, int, float, bool)): + normalised[key] = value + elif value is None: + normalised[key] = "null" + elif isinstance(value, (list, dict)): + normalised[key] = json.dumps(value, ensure_ascii=False, sort_keys=True) + else: + normalised[key] = str(value) + return normalised + +def _normalise_record(record: object, line_number: int) -> dict: + """Validate one archive JSONL record and return a Chroma-ready mapping.""" + if not isinstance(record, dict): + raise ValueError(f"Archive record on line {line_number} must be an object") + + record_id = record.get("id") + document = record.get("document") + embedding = record.get("embedding", []) + if not isinstance(record_id, str) or not record_id: + raise ValueError(f"Archive record on line {line_number} has an invalid id") + if not isinstance(document, str): + raise ValueError(f"Archive record {record_id!r} has a non-string document") + if not isinstance(embedding, list) or not all( + isinstance(value, (int, float)) and not isinstance(value, bool) + for value in embedding): + raise ValueError(f"Archive record {record_id!r} has an invalid embedding") + + return { + "id": record_id, + "document": document, + "embedding": embedding, + "metadata": _normalise_metadata(record.get("metadata", {})), + } + +def _record_batches(records_path: Path): + """Yield validated archive records in bounded JSONL batches.""" + batch = [] + with records_path.open("r", encoding="utf-8") as records_file: + for line_number, line in enumerate(records_file, 1): + if not line.strip(): + continue + try: + raw_record = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"Invalid JSONL record on line {line_number}: {exc.msg}" + ) from exc + batch.append(_normalise_record(raw_record, line_number)) + if len(batch) == _VECTOR_BATCH: + yield batch + batch = [] + if batch: + yield batch + +def _validate_manifest(manifest: object) -> dict: + if not isinstance(manifest, dict): + raise ValueError("Archive manifest must be an object") + if type(manifest.get("format_version")) is not int: + raise ValueError("Archive manifest has invalid format_version") + for field in ("omegaclaw_version", "chromadb_version", "created_at"): + if not isinstance(manifest.get(field), str) or not manifest[field]: + raise ValueError(f"Archive manifest has invalid {field}") + try: + created_at = datetime.fromisoformat(manifest["created_at"].replace("Z", "+00:00")) + except ValueError as exc: + raise ValueError("Archive manifest has invalid created_at") from exc + if created_at.tzinfo is None or created_at.utcoffset() != timezone.utc.utcoffset(created_at): + raise ValueError("Archive manifest created_at must be UTC") + components = manifest.get("components") + if (not isinstance(components, list) + or any(component not in ("history", "ltm") for component in components) + or len(components) != len(set(components))): + raise ValueError("Archive manifest has invalid components") + if not isinstance(manifest.get("record_count"), int) or isinstance( + manifest["record_count"], bool) or manifest["record_count"] < 0: + raise ValueError("Archive manifest has invalid record_count") + if not isinstance(manifest.get("history_bytes"), int) or isinstance( + manifest["history_bytes"], bool) or manifest["history_bytes"] < 0: + raise ValueError("Archive manifest has invalid history_bytes") + if "ltm" not in components and manifest["record_count"]: + raise ValueError("Archive manifest has records without the ltm component") + if "history" not in components and manifest["history_bytes"]: + raise ValueError("Archive manifest has history bytes without the history component") + checksums = manifest.get("checksums") + if not isinstance(checksums, dict) or not all( + isinstance(name, str) and isinstance(value, str) + and len(value) == 64 and all(char in "0123456789abcdef" for char in value.lower()) + for name, value in checksums.items()): + raise ValueError("Archive manifest has invalid checksums") + expected_files = set().union(*(_COMPONENT_FILES[component] for component in components)) + if set(checksums) != expected_files: + raise ValueError("Archive manifest checksums do not match its components") + if "ltm" in components: + _validate_embedding_info(manifest) + elif manifest.get("embedding_info") != {}: + raise ValueError("Archive manifest has embedding_info without the ltm component") + return manifest + +def _validate_embedding_info(manifest: dict) -> None: + embedding_info = manifest.get("embedding_info") + if (not isinstance(embedding_info, dict) + or not isinstance(embedding_info.get("provider"), str) + or not isinstance(embedding_info.get("model"), str) + or not isinstance(embedding_info.get("vector_dimension"), int) + or isinstance(embedding_info["vector_dimension"], bool) + or embedding_info["vector_dimension"] < 0): + raise ValueError("Archive manifest has invalid embedding_info") + +def _validate_records(records_path: Path, manifest: dict) -> None: + expected_count = manifest["record_count"] + dimension = manifest["embedding_info"]["vector_dimension"] + count = 0 + for records in _record_batches(records_path): + for record in records: + if record["embedding"] and len(record["embedding"]) != dimension: + raise ValueError("Archive record embedding dimension does not match manifest") + count += 1 + if count != expected_count: + raise ValueError("Archive record count does not match manifest") + +def _validate_extracted_archive(staging: Path, manifest: dict) -> None: + """Validate extracted content before import can modify live memory.""" + for member_name, expected in manifest["checksums"].items(): + actual = _sha256(staging / member_name) + if actual != expected: + raise ValueError(f"Checksum mismatch for {member_name!r}") + + components = manifest["components"] + if "history" in components: + history = staging / "history" / "history.metta" + if history.stat().st_size != manifest["history_bytes"]: + raise ValueError("Archive history size does not match manifest") + if "ltm" in components: + collections = json.loads( + (staging / "vector" / "collections.json").read_text(encoding="utf-8") + ) + if (not isinstance(collections, dict) + or collections.get("name") != "memories" + or collections.get("embedding_info") != manifest["embedding_info"]): + raise ValueError("Archive collections metadata does not match manifest") + _validate_records(staging / "vector" / "records.jsonl", manifest) + +def _collection_dimension(collection) -> int | None: + result = collection.get(limit=1, include=["embeddings"]) + embeddings = result.get("embeddings") + return len(embeddings[0]) if embeddings is not None and len(embeddings) else None + +def _staging_dir() -> Path: + """Return same-filesystem staging for atomic archive publication.""" + staging = TRANSFER_DIR / ".staging" + staging.mkdir(parents=True, exist_ok=True) + return staging + +def _is_user_memory_record(metadata: object) -> bool: + """Return whether a collection record belongs to portable user memory.""" + if not isinstance(metadata, dict): + return False + return ( + metadata.get("record_kind") == gateway._RECORD_KIND + or ("record_kind" not in metadata and "type" not in metadata) + ) + +def _export_history(staging: Path) -> int: + """Copy history.metta into staging. Returns byte size.""" + src = history_path() + dst = staging / "history" / "history.metta" + dst.parent.mkdir(parents=True, exist_ok=True) + if src.exists(): + shutil.copy2(src, dst) + return dst.stat().st_size + dst.touch() + return 0 + +def _export_vectors(staging: Path) -> tuple[int, dict]: + """Export marked and legacy user records into staging.""" + vector_dir = staging / "vector" + vector_dir.mkdir(parents=True, exist_ok=True) + records_path = vector_dir / "records.jsonl" + col = gateway._get_collection() + record_count = 0 + dimension = 0 + + with records_path.open("w", encoding="utf-8") as records_file: + for offset in range(0, col.count(), _VECTOR_BATCH): + result = col.get( + limit=_VECTOR_BATCH, + offset=offset, + include=["documents", "metadatas", "embeddings"], + ) + ids = result.get("ids", []) + docs = result.get("documents") or [] + metas = result.get("metadatas") or [] + raw_embeddings = result.get("embeddings") + embeddings = list(raw_embeddings) if raw_embeddings is not None else [] + for index, record_id in enumerate(ids): + metadata = metas[index] if index < len(metas) else {} + if not _is_user_memory_record(metadata): + continue + embedding = embeddings[index] if index < len(embeddings) else [] + embedding = list(embedding) if hasattr(embedding, "__iter__") else embedding + if not dimension and embedding: + dimension = len(embedding) + records_file.write(json.dumps({ + "id": record_id, + "document": docs[index] if index < len(docs) else "", + "metadata": _normalise_metadata(metadata), + "embedding": embedding, + }, ensure_ascii=False) + "\n") + record_count += 1 + + active_profile = _embedding_profile() + embedding_info = { + "provider": active_profile["provider"], + "model": active_profile["model"], + "vector_dimension": dimension, + } + + (vector_dir / "collections.json").write_text( + json.dumps({"name": "memories", "embedding_info": embedding_info}, indent=2), + encoding="utf-8", + ) + + return record_count, embedding_info + +def _build_manifest(staging: Path, components: list[str], + record_count: int, embedding_info: dict) -> None: + """Write manifest.json into staging.""" + from src.helper import omegaclaw_version + + checksums: dict[str, str] = {} + for member in _ALLOWLIST: + p = staging / member + if p.exists(): + checksums[member] = _sha256(p) + + manifest = { + "format_version": ARCHIVE_FORMAT_VERSION, + "omegaclaw_version": omegaclaw_version(), + "chromadb_version": chromadb.__version__, + "components": components, + "embedding_info": embedding_info, + "record_count": record_count, + "history_bytes": (staging / "history" / "history.metta").stat().st_size + if (staging / "history" / "history.metta").exists() else 0, + "created_at": _utc_now(), + "checksums": checksums, + } + (staging / "manifest.json").write_text( + json.dumps(manifest, indent=2), encoding="utf-8" + ) + +def _pack_archive(staging: Path, dest: Path) -> None: + """Pack staging directory into a .tar.gz at dest.""" + with tarfile.open(dest, "w:gz") as tar: + for member in sorted(_ALLOWLIST): + p = staging / member + if p.exists(): + tar.add(p, arcname=member) + +def start_export_job( + component: str, on_complete: Callable[[str, dict], None] | None = None +) -> str: + """Start an asynchronous export and return its job ID.""" + if component not in ("history", "ltm", "both"): + raise ValueError(f"Invalid component: {component!r}. Use history, ltm, or both.") + + job_id = uuid.uuid4().hex + with _jobs_lock: + _jobs[job_id] = {"status": "running"} + threading.Thread( + target=_run_export, args=(job_id, component, on_complete), daemon=True + ).start() + logger.info(f"memory_transfer: export job {job_id} started (component={component})") + return job_id + +def _run_export( + job_id: str, component: str, on_complete: Callable[[str, dict], None] | None +) -> None: + try: + result = export(component) + status = {"status": "done", **result} + except Exception as exc: + logger.exception(f"memory_transfer: export job {job_id} failed: {exc}") + status = {"status": "failed", "error": str(exc)} + + with _jobs_lock: + _jobs[job_id] = status + + if on_complete is not None: + try: + on_complete(job_id, status.copy()) + except Exception as exc: + logger.exception( + f"memory_transfer: completion callback for export job {job_id} failed: {exc}" + ) + +_jobs: dict[str, dict] = {} +_jobs_lock = threading.Lock() + +def get_export_status(job_id: str) -> dict: + with _jobs_lock: + return _jobs.get(job_id, {"status": "unknown"}).copy() + +def export(component: str) -> dict: + """Export selected memory components and publish an archive atomically.""" + include_history = component in ("history", "both") + include_vectors = component in ("ltm", "both") + + archive_name = _archive_name() + work_dir = _staging_dir() / archive_name + work_dir.mkdir(parents=True, exist_ok=True) + staging = work_dir / "staging" + staging.mkdir() + # Landlock permits atomic rename within a directory, but can reject a + # rename between .staging and its parent as a cross-directory operation. + tmp_archive = TRANSFER_DIR / f".{archive_name}.tmp" + + try: + record_count = 0 + embedding_info: dict = {} + components: list[str] = [] + + with gateway._write_lock: + if include_history: + _export_history(staging) + components.append("history") + if include_vectors: + record_count, embedding_info = _export_vectors(staging) + components.append("ltm") + + _build_manifest(staging, components, record_count, embedding_info) + _pack_archive(staging, tmp_archive) + _verify_archive(tmp_archive) + + TRANSFER_DIR.mkdir(parents=True, exist_ok=True) + dest = TRANSFER_DIR / archive_name + os.replace(tmp_archive, dest) + + finally: + tmp_archive.unlink(missing_ok=True) + shutil.rmtree(work_dir, ignore_errors=True) + + size = dest.stat().st_size + checksum = _sha256(dest) + logger.info(f"memory_transfer: exported {dest} ({size} bytes, sha256={checksum})") + return { + "filename": archive_name, + "size": size, + "checksum": checksum, + "record_count": record_count, + "components": components, + } + +def _verify_archive(path: Path, extract_to: Path | None = None) -> dict: + """Validate archive members and extract once for checksum verification.""" + if path.stat().st_size > _MAX_COMPRESSED_BYTES: + raise ValueError(f"Archive too large: {path.stat().st_size} bytes") + + seen_names: set[str] = set() + total_extracted = 0 + + with tarfile.open(path, "r:gz") as tar: + for member in tar.getmembers(): + name = member.name + if name not in _ALLOWLIST: + raise ValueError(f"Unexpected archive member: {name!r}") + if name in seen_names: + raise ValueError(f"Duplicate archive member: {name!r}") + seen_names.add(name) + if not member.isfile(): + raise ValueError(f"Non-regular member: {name!r}") + if ".." in Path(name).parts or Path(name).is_absolute(): + raise ValueError(f"Path traversal in member: {name!r}") + total_extracted += member.size + if total_extracted > _MAX_EXTRACTED_BYTES: + raise ValueError("Archive extracted size exceeds limit") + + manifest_file = tar.extractfile("manifest.json") + if manifest_file is None: + raise ValueError("manifest.json missing from archive") + manifest = _validate_manifest(json.loads(manifest_file.read())) + + if manifest["format_version"] != ARCHIVE_FORMAT_VERSION: + raise ValueError(f"Unsupported format_version: {manifest.get('format_version')}") + + components = manifest.get("components", []) + expected_members = {"manifest.json"} + expected_members.update(*(_COMPONENT_FILES[component] for component in components)) + if seen_names != expected_members: + raise ValueError("Archive members do not match manifest components") + + if extract_to is not None: + extract_to.mkdir(parents=True, exist_ok=True) + _safe_extract(tar, extract_to) + _validate_extracted_archive(extract_to, manifest) + else: + with tempfile.TemporaryDirectory() as tmp: + staging = Path(tmp) + _safe_extract(tar, staging) + _validate_extracted_archive(staging, manifest) + + return manifest + +def _safe_extract(tar: tarfile.TarFile, dest: Path) -> None: + """Extract only regular, allowlisted archive members without tarfile filters. + + Python 3.11 does not support TarFile.extractall(filter=...), so extraction + is performed explicitly after validating each member's fixed archive path. + """ + base = dest.resolve() + for member in tar.getmembers(): + name = member.name + if name not in _ALLOWLIST or not member.isfile(): + raise ValueError(f"Unsafe archive member: {name!r}") + target = (base / name).resolve() + if base not in target.parents: + raise ValueError(f"Path traversal in member: {name!r}") + target.parent.mkdir(parents=True, exist_ok=True) + source = tar.extractfile(member) + if source is None: + raise ValueError(f"Could not read archive member: {name!r}") + with source, target.open("wb") as output: + shutil.copyfileobj(source, output) + +def _parse_component_flags(args: argparse.Namespace) -> tuple[bool, bool]: + no_history = getattr(args, "no_history", False) + no_vector = getattr(args, "no_vector", False) or getattr(args, "only_history", False) + if no_history and no_vector: + raise ValueError("--no-history and --no-vector together import nothing. Aborting.") + return not no_history, not no_vector + +def _tx_marker(memory_base: Path) -> Path: + return memory_base / _TX_MARKER_NAME + +def _receipt_path(memory_base: Path, digest: str, mode: str, + include_history: bool, include_vectors: bool) -> Path: + components = "-".join(component for component, included in ( + ("history", include_history), ("ltm", include_vectors) + ) if included) + return memory_base / _RECEIPT_DIR_NAME / f"{digest}-{mode}-{components}.json" + +def _write_receipt(path: Path, digest: str, mode: str, + include_history: bool, include_vectors: bool) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + receipt = { + "archive_sha256": digest, + "mode": mode, + "include_history": include_history, + "include_vectors": include_vectors, + "imported_at": _utc_now(), + } + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(receipt, sort_keys=True), encoding="utf-8") + os.replace(temporary, path) + +def _marker_has_receipt(marker: Path, memory_base: Path) -> bool: + try: + receipt_name = json.loads(marker.read_text(encoding="utf-8")).get("receipt") + except (json.JSONDecodeError, OSError): + return False + return ( + isinstance(receipt_name, str) + and Path(receipt_name).name == receipt_name + and (memory_base / _RECEIPT_DIR_NAME / receipt_name).is_file() + ) + +def _append_state(include_history: bool, include_vectors: bool) -> dict: + history = history_path() + return { + "history": { + "existed": history.exists(), + "size": history.stat().st_size if history.exists() else 0, + } if include_history else None, + "import_id": uuid.uuid4().hex if include_vectors else None, + } + +def _rollback_append(state: dict) -> None: + history = state.get("history") + if isinstance(history, dict): + path = history_path() + if history.get("existed"): + if path.exists(): + with path.open("rb+") as output: + output.truncate(history["size"]) + else: + path.unlink(missing_ok=True) + + import_id = state.get("import_id") + if isinstance(import_id, str): + col = gateway._get_collection() + imported = col.get(where={"import_id": import_id}, include=[]) + if imported["ids"]: + col.delete(ids=imported["ids"]) + +def recover(memory_base: Path | None = None) -> None: + """Restore an interrupted import or fail when its rollback is unavailable.""" + base = memory_base or memory_dir_path() + marker = _tx_marker(base) + rollback = base / ".import_rollback" + + if not marker.exists(): + return + + if _marker_has_receipt(marker, base): + marker.unlink(missing_ok=True) + shutil.rmtree(rollback, ignore_errors=True) + logger.info("memory_transfer: completed import transaction recovered") + return + + try: + transaction = json.loads(marker.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + transaction = {} + append = transaction.get("append") if isinstance(transaction, dict) else None + if isinstance(append, dict): + logger.warning("memory_transfer: unfinished append import detected — removing partial import") + _rollback_append(append) + marker.unlink(missing_ok=True) + logger.info("memory_transfer: append transaction recovery complete") + return + + logger.warning("memory_transfer: unfinished import transaction detected — restoring rollback") + + if not rollback.exists(): + raise RuntimeError( + "Import transaction marker found but no rollback copy exists. " + "Cannot recover safely — operator intervention required." + ) + + _restore_rollback(rollback) + marker.unlink(missing_ok=True) + shutil.rmtree(rollback, ignore_errors=True) + logger.info("memory_transfer: transaction recovery complete") + +def _restore_rollback(rollback: Path) -> None: + hist_rb = rollback / "history.metta" + chroma_rb = rollback / "chroma_db" + state_path = rollback / _ROLLBACK_STATE_NAME + try: + state = json.loads(state_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + state = {} + + if state.get("history") is False: + history_path().unlink(missing_ok=True) + elif hist_rb.exists(): + history_path().parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(hist_rb, history_path()) + + live_chroma = chroma_db_path() + if state.get("vectors") is False: + shutil.rmtree(live_chroma, ignore_errors=True) + elif chroma_rb.exists(): + if live_chroma.exists(): + shutil.rmtree(live_chroma) + shutil.copytree(chroma_rb, live_chroma) + if "vectors" in state: + gateway._client = None + gateway._collection = None + +def import_archive(archive_path: Path, mode: str = "overwrite", + include_history: bool = True, + include_vectors: bool = True) -> None: + """Validate and restore a memory archive before the agent loop starts.""" + if mode not in ("overwrite", "append"): + raise ValueError(f"Invalid mode: {mode!r}. Use overwrite or append.") + if not archive_path.exists(): + raise FileNotFoundError(f"Archive not found: {archive_path}") + + logger.info(f"memory_transfer: importing {archive_path} (mode={mode})") + + base = memory_dir_path() + digest = _sha256(archive_path) + receipt = _receipt_path(base, digest, mode, include_history, include_vectors) + if receipt.exists(): + logger.info("memory_transfer: archive already imported; skipping") + return + staging = base / ".import_staging" + shutil.rmtree(staging, ignore_errors=True) + try: + manifest = _verify_archive(archive_path, staging) + import_history = ( + include_history + and "history" in manifest.get("components", []) + and (staging / "history" / "history.metta").is_file() + ) + import_vectors = ( + include_vectors + and "ltm" in manifest.get("components", []) + and (staging / "vector" / "records.jsonl").is_file() + ) + if mode == "overwrite": + _import_overwrite( + staging, manifest, import_history, import_vectors, receipt, digest + ) + else: + _import_append( + staging, manifest, import_history, import_vectors, receipt, digest + ) + finally: + shutil.rmtree(staging, ignore_errors=True) + + logger.info("memory_transfer: import complete") + +def _import_overwrite(staging: Path, manifest: dict, + include_history: bool, include_vectors: bool, + receipt: Path, digest: str) -> None: + """Overwrite live memory with rollback and crash-recovery protection.""" + base = memory_dir_path() + rollback = base / ".import_rollback" + marker = _tx_marker(base) + + shutil.rmtree(rollback, ignore_errors=True) + rollback.mkdir(parents=True) + state = {"history": history_path().exists() if include_history else None, + "vectors": chroma_db_path().exists() if include_vectors else None} + (rollback / _ROLLBACK_STATE_NAME).write_text( + json.dumps(state), encoding="utf-8" + ) + if include_history and state["history"]: + shutil.copy2(history_path(), rollback / "history.metta") + if include_vectors and state["vectors"]: + rb_chroma = rollback / "chroma_db" + shutil.copytree(chroma_db_path(), rb_chroma) + + marker.write_text(json.dumps({"receipt": receipt.name}), encoding="utf-8") + try: + if include_history: + src = staging / "history" / "history.metta" + history_path().parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, history_path()) + + if include_vectors: + _restore_vectors(staging, manifest) + + _smoke_test(include_history, include_vectors) + + except Exception: + logger.exception("memory_transfer: overwrite failed — restoring rollback") + try: + _restore_rollback(rollback) + except Exception: + logger.exception("memory_transfer: rollback itself failed — preserving marker and rollback for manual recovery") + raise + marker.unlink(missing_ok=True) + shutil.rmtree(rollback, ignore_errors=True) + raise + else: + _write_receipt(receipt, digest, "overwrite", include_history, include_vectors) + marker.unlink(missing_ok=True) + shutil.rmtree(rollback, ignore_errors=True) + +def _restore_vectors(staging: Path, manifest: dict) -> None: + """Restore user-memory vectors while preserving non-user records.""" + records_path = staging / "vector" / "records.jsonl" + if not records_path.exists(): + return + + col = gateway._get_collection() + target_dimension = _collection_dimension(col) + + existing = col.get(include=["metadatas"]) + user_ids = [ + eid for eid, meta in zip(existing["ids"], existing.get("metadatas") or []) + if _is_user_memory_record(meta) + ] + if user_ids: + col.delete(ids=user_ids) + + gateway._client = None + gateway._collection = None + col = gateway._get_collection() + + for records in _record_batches(records_path): + _reembed_records(records, manifest, target_dimension) + col.add( + ids =[record["id"] for record in records], + documents =[record["document"] for record in records], + embeddings =[record["embedding"] for record in records], + metadatas =[record["metadata"] for record in records], + ) + +def _import_append(staging: Path, manifest: dict, + include_history: bool, include_vectors: bool, + receipt: Path, digest: str) -> None: + """Append imported memory to existing live memory.""" + base = memory_dir_path() + marker = _tx_marker(base) + state = _append_state(include_history, include_vectors) + marker.write_text( + json.dumps({"receipt": receipt.name, "append": state}), encoding="utf-8" + ) + try: + if include_history: + src = staging / "history" / "history.metta" + gateway.append_history("\n" + src.read_text(encoding="utf-8")) + + if include_vectors: + records_path = staging / "vector" / "records.jsonl" + col = gateway._get_collection() + target_dimension = _collection_dimension(col) + import_id = state["import_id"] + for records in _record_batches(records_path): + _reembed_records(records, manifest, target_dimension) + col.upsert( + ids =[f"import-{import_id}-{record['id']}" for record in records], + documents =[record["document"] for record in records], + embeddings=[record["embedding"] for record in records], + metadatas=[{**record["metadata"], "import_id": import_id} + for record in records], + ) + _smoke_test(include_history, include_vectors) + except Exception: + logger.exception("memory_transfer: append failed — removing partial import") + try: + _rollback_append(state) + except Exception: + logger.exception("memory_transfer: append cleanup failed — preserving marker for recovery") + raise + marker.unlink(missing_ok=True) + raise + else: + _write_receipt(receipt, digest, "append", include_history, include_vectors) + marker.unlink(missing_ok=True) + +def _smoke_test(include_history: bool, include_vectors: bool) -> None: + if include_history: + history_path().read_text(encoding="utf-8")[:1] + if include_vectors: + gateway._get_collection().get(limit=1) + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="memory_transfer") + sub = parser.add_subparsers(dest="command", required=True) + + imp = sub.add_parser("import", help="Restore a memory archive (pre-start)") + imp.add_argument("archive", type=Path) + imp.add_argument("--mode", choices=["overwrite", "append"], default="overwrite") + imp.add_argument("--no-history", action="store_true") + imp.add_argument("--no-vector", action="store_true") + imp.add_argument("--only-history", action="store_true") + + sub.add_parser("recover", help="Recover from an interrupted import transaction") + return parser + +if __name__ == "__main__": + import sys + args = _build_parser().parse_args() + + if args.command == "recover": + recover() + sys.exit(0) + try: + inc_hist, inc_vec = _parse_component_flags(args) + import_archive(args.archive, mode=args.mode, + include_history=inc_hist, include_vectors=inc_vec) + except Exception as exc: + logger.error(f"memory_transfer import failed: {exc}") + sys.exit(1) diff --git a/tests/test_memory_transfer.py b/tests/test_memory_transfer.py new file mode 100644 index 00000000..ed82e2c6 --- /dev/null +++ b/tests/test_memory_transfer.py @@ -0,0 +1,273 @@ +import importlib +import io +import json +import tarfile +import threading +import types +from pathlib import Path + +import pytest +import yaml + +EMBEDDING = [0.1, 0.2, 0.3] +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture(autouse=True) +def isolated_memory(tmp_path, monkeypatch): + memory_dir = tmp_path / "memory" + memory_dir.mkdir() + monkeypatch.setenv("MEMORY_DIR", str(memory_dir)) + + import src.memory_layout as layout + importlib.reload(layout) + import src.memory_gateway as gateway + importlib.reload(gateway) + import src.memory_transfer as transfer + importlib.reload(transfer) + transfer.TRANSFER_DIR = tmp_path / "transfer" + transfer.TRANSFER_DIR.mkdir() + return transfer, gateway, memory_dir + + +def test_launcher_and_policy_wiring(): + launcher = (REPO_ROOT / "scripts" / "omegaclaw").read_text() + entrypoint = (REPO_ROOT / "entrypoint.sh").read_text() + config = (REPO_ROOT / "config" / "config.yaml").read_text() + dockerfile = (REPO_ROOT / "Dockerfile").read_text() + policy = yaml.safe_load((REPO_ROOT / "profile" / "policy.yaml").read_text()) + + assert "memoryExportEnabled: false" in config + assert "--enable-memory-export" in launcher + assert "MEMORY_IMPORT_ONLY_HISTORY" in launcher + assert ' -- sh "$OMEGACLAW_DIR" "${import_args[@]}"' in entrypoint + assert 'echo "memory_transfer: import complete"' in entrypoint + assert "mkdir -p /memory-transfer" in dockerfile + assert "/memory-transfer" in policy["filesystem_policy"]["read_write"] + + +def test_export_both_contains_only_portable_user_memory(isolated_memory): + transfer, gateway, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("portable history\n") + gateway.remember("portable fact", EMBEDDING, "2026-01-01") + gateway._get_collection().add( + ids=["knowledge"], documents=["non-user data"], embeddings=[EMBEDDING], + metadatas=[{"type": "chunk"}], + ) + + result = transfer.export("both") + with tarfile.open(transfer.TRANSFER_DIR / result["filename"], "r:gz") as archive: + assert set(archive.getnames()) == { + "manifest.json", "history/history.metta", "vector/collections.json", "vector/records.jsonl" + } + records = [json.loads(line) for line in archive.extractfile("vector/records.jsonl")] + + assert result["record_count"] == 1 + assert [record["document"] for record in records] == ["portable fact"] + + +def test_concurrent_exports_publish_distinct_archives(isolated_memory): + transfer, _, _ = isolated_memory + results = [] + threads = [threading.Thread(target=lambda: results.append(transfer.export("history"))) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + + assert all(not thread.is_alive() for thread in threads) + assert len({result["filename"] for result in results}) == 2 + + +def test_malformed_archive_is_rejected_before_mutating_memory(isolated_memory, tmp_path): + transfer, _, memory_dir = isolated_memory + history = memory_dir / "history.metta" + history.write_text("original\n") + valid = transfer.TRANSFER_DIR / transfer.export("history")["filename"] + invalid = tmp_path / "invalid.tar.gz" + with tarfile.open(valid, "r:gz") as source, tarfile.open(invalid, "w:gz") as output: + for member in source.getmembers(): + output.addfile(member, source.extractfile(member)) + extra = tarfile.TarInfo("unexpected.txt") + extra.size = 1 + output.addfile(extra, io.BytesIO(b"x")) + + history.write_text("live\n") + with pytest.raises(ValueError, match="Unexpected archive member"): + transfer.import_archive(invalid) + assert history.read_text() == "live\n" + + +def test_invalid_record_count_is_rejected_before_mutating_vectors(isolated_memory, tmp_path): + transfer, gateway, _ = isolated_memory + gateway.remember("source fact", EMBEDDING, "2026-01-01") + valid = transfer.TRANSFER_DIR / transfer.export("ltm")["filename"] + invalid = tmp_path / "invalid-count.tar.gz" + with tarfile.open(valid, "r:gz") as source, tarfile.open(invalid, "w:gz") as output: + for member in source.getmembers(): + if member.name == "manifest.json": + manifest = json.loads(source.extractfile(member).read()) + manifest["record_count"] = 2 + data = json.dumps(manifest).encode() + member.size = len(data) + output.addfile(member, io.BytesIO(data)) + else: + output.addfile(member, source.extractfile(member)) + + with pytest.raises(ValueError, match="record count"): + transfer.import_archive(invalid, include_history=False, include_vectors=True) + assert gateway._get_collection().count() == 1 + + +def test_invalid_manifest_schema_is_rejected_before_mutating_memory(isolated_memory, tmp_path): + transfer, _, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("archive history\n") + valid = transfer.TRANSFER_DIR / transfer.export("history")["filename"] + invalid = tmp_path / "invalid-manifest.tar.gz" + with tarfile.open(valid, "r:gz") as source, tarfile.open(invalid, "w:gz") as output: + for member in source.getmembers(): + if member.name == "manifest.json": + manifest = json.loads(source.extractfile(member).read()) + manifest["created_at"] = 1 + data = json.dumps(manifest).encode() + member.size = len(data) + output.addfile(member, io.BytesIO(data)) + else: + output.addfile(member, source.extractfile(member)) + + (memory_dir / "history.metta").write_text("live history\n") + with pytest.raises(ValueError, match="created_at"): + transfer.import_archive(invalid) + assert (memory_dir / "history.metta").read_text() == "live history\n" + + +def test_import_extracts_archive_once(isolated_memory, monkeypatch): + transfer, _, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("archive history\n") + archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] + extractions = [] + original_extract = transfer._safe_extract + + def track_extract(tar, dest): + extractions.append(dest) + original_extract(tar, dest) + + monkeypatch.setattr(transfer, "_safe_extract", track_extract) + transfer.import_archive(archive) + + assert len(extractions) == 1 + + +def test_overwrite_round_trip_restores_history_and_ltm(isolated_memory): + transfer, gateway, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("archived history\n") + gateway.remember("archived fact", EMBEDDING, "2026-01-01") + collection = gateway._get_collection() + collection.add( + ids=["knowledge"], documents=["retain this"], embeddings=[EMBEDDING], metadatas=[{"type": "chunk"}] + ) + archive = transfer.TRANSFER_DIR / transfer.export("both")["filename"] + + (memory_dir / "history.metta").write_text("live history\n") + collection.delete(ids=[record_id for record_id in collection.get(include=[])["ids"] if record_id != "knowledge"]) + transfer.import_archive(archive, mode="overwrite") + + documents = gateway._get_collection().get(include=["documents"])["documents"] + assert (memory_dir / "history.metta").read_text() == "archived history\n" + assert set(documents) == {"archived fact", "retain this"} + + +def test_history_only_import_does_not_open_chromadb(isolated_memory, monkeypatch): + transfer, gateway, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("archived history\n") + archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] + monkeypatch.setattr(gateway, "_get_collection", lambda: pytest.fail("opened ChromaDB")) + + transfer.import_archive(archive, include_history=True, include_vectors=False) + + assert (memory_dir / "history.metta").read_text() == "archived history\n" + + +def test_reembedding_runs_for_changed_embedding_profile(isolated_memory, monkeypatch): + transfer, gateway, _ = isolated_memory + gateway.remember("portable fact", EMBEDDING, "2026-01-01") + archive = transfer.TRANSFER_DIR / transfer.export("ltm")["filename"] + monkeypatch.setattr(transfer, "_embedding_profile", lambda: {"provider": "Local", "model": "new"}) + rag = types.ModuleType("src.rag") + rag.local_embed_batch = lambda documents: [[0.4, 0.5, 0.6] for _ in documents] + rag.openai_embed_batch = rag.local_embed_batch + monkeypatch.setitem(__import__("sys").modules, "src.rag", rag) + + transfer.import_archive(archive, include_history=False, include_vectors=True) + + embedding = gateway._get_collection().get(include=["embeddings"])["embeddings"][0] + assert list(embedding) == pytest.approx([0.4, 0.5, 0.6]) + + +def test_overwrite_failure_restores_absent_history_state(isolated_memory, monkeypatch): + transfer, _, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("archive history\n") + archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] + (memory_dir / "history.metta").unlink() + monkeypatch.setattr(transfer, "_smoke_test", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) + + with pytest.raises(RuntimeError, match="boom"): + transfer.import_archive(archive) + assert not (memory_dir / "history.metta").exists() + + +def test_receipt_prevents_repeat_overwrite(isolated_memory): + transfer, _, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("archive history\n") + archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] + (memory_dir / "history.metta").write_text("before import\n") + transfer.import_archive(archive) + (memory_dir / "history.metta").write_text("new memory\n") + + transfer.import_archive(archive) + + assert (memory_dir / "history.metta").read_text() == "new memory\n" + + +def test_append_failure_removes_partial_history_and_vectors(isolated_memory, monkeypatch): + transfer, gateway, memory_dir = isolated_memory + (memory_dir / "history.metta").write_text("archive history\n") + gateway.remember("archive fact", EMBEDDING, "2026-01-01") + archive = transfer.TRANSFER_DIR / transfer.export("both")["filename"] + (memory_dir / "history.metta").write_text("live history\n") + gateway.remember("live fact", EMBEDDING, "2026-01-02") + original_count = gateway._get_collection().count() + monkeypatch.setattr(transfer, "_smoke_test", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) + + with pytest.raises(RuntimeError, match="boom"): + transfer.import_archive(archive, mode="append") + + assert (memory_dir / "history.metta").read_text() == "live history\n" + assert gateway._get_collection().count() == original_count + + +def test_recovery_removes_interrupted_append(isolated_memory): + transfer, gateway, memory_dir = isolated_memory + history = memory_dir / "history.metta" + history.write_text("original\npartial\n") + import_id = "interrupted" + gateway._get_collection().add( + ids=["partial"], documents=["partial fact"], embeddings=[EMBEDDING], + metadatas=[{"import_id": import_id}], + ) + (memory_dir / transfer._TX_MARKER_NAME).write_text(json.dumps({"append": { + "history": {"existed": True, "size": len("original\n")}, "import_id": import_id, + }})) + + transfer.recover(memory_dir) + + assert history.read_text() == "original\n" + assert gateway._get_collection().get(where={"import_id": import_id}, include=[])["ids"] == [] + + +def test_export_is_disabled_until_explicitly_enabled(isolated_memory, monkeypatch): + transfer, _, _ = isolated_memory + monkeypatch.delenv("OMEGACLAW_memoryExportEnabled", raising=False) + assert transfer.is_export_enabled() is False + monkeypatch.setenv("OMEGACLAW_memoryExportEnabled", "true") + assert transfer.is_export_enabled() is True From ed844a66c6c50aa1dd878d7fddacc33eeaf37ea4 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Mon, 17 Aug 2026 19:59:02 +0300 Subject: [PATCH 03/34] feat(launcher): add memory transfer startup options --- Dockerfile | 3 + entrypoint.sh | 34 +++++++ profile/policy.yaml | 1 + scripts/omegaclaw | 216 +++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 242 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index d297c58c..d43eb119 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,11 +134,14 @@ COPY --from=versioned-source /omegaclaw-source ${OMEGACLAW_DIR} RUN cp ${OMEGACLAW_DIR}/run.metta /PeTTa/run.metta \ && mkdir -p ${MEMORY_DIR}/chroma_db \ + && mkdir -p /memory-transfer \ && ln -s ${MEMORY_DIR}/chroma_db ./chroma_db \ && chmod +x ${OMEGACLAW_DIR}/entrypoint.sh \ && chmod +x ${OMEGACLAW_DIR}/scripts/import_knowledge.sh \ && chmod +x ${OMEGACLAW_DIR}/scripts/omegaclaw \ && chown -R 65534:65534 ${MEMORY_DIR} \ + && chown 65534:65534 /memory-transfer \ + && chmod 0700 /memory-transfer \ && find ${MEMORY_DIR} -type f -exec chmod 0644 {} \; \ && chmod 0444 ${MEMORY_DIR}/prompt.txt \ && chown -R 65534:65534 /opt/huggingface /opt/sentence_transformers diff --git a/entrypoint.sh b/entrypoint.sh index 6bb54133..bc92aa7d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -40,10 +40,44 @@ if [[ "${IMPORT_KB_ON_START}" == "1" ]]; then su nobody -s /bin/sh -c "${OMEGACLAW_DIR}/scripts/import_knowledge.sh" fi +# Verify that the agent user can write the mounted transfer directory. +if [[ "${MEMORY_TRANSFER_MOUNTED:-0}" == "1" ]]; then + su nobody -s /bin/sh -c 'test -d /memory-transfer && test -w /memory-transfer' \ + || { echo "Memory transfer directory is not writable by the agent user." >&2; exit 1; } +fi + +# Recover an interrupted import before starting the agent. +su nobody -s /bin/sh -c 'cd "$1" && exec python3 -m src.memory_transfer recover' \ + sh "$OMEGACLAW_DIR" \ + || { echo "Memory import recovery failed. Aborting startup." >&2; exit 1; } + +# Validate the archive filename again at the container boundary. +if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then + if [[ ! "${MEMORY_IMPORT_FILE}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.tar\.gz$ ]]; then + echo "MEMORY_IMPORT_FILE must be a plain filename, not a path: ${MEMORY_IMPORT_FILE}" >&2 + exit 1 + fi + case "${MEMORY_IMPORT_MODE:-overwrite}" in + overwrite|append) ;; + *) echo "MEMORY_IMPORT_MODE must be overwrite or append" >&2; exit 1 ;; + esac + import_args=("/memory-transfer/${MEMORY_IMPORT_FILE}" --mode "${MEMORY_IMPORT_MODE:-overwrite}") + [[ "${MEMORY_IMPORT_NO_HISTORY:-0}" == "1" ]] && import_args+=(--no-history) + [[ "${MEMORY_IMPORT_NO_VECTOR:-0}" == "1" ]] && import_args+=(--no-vector) + [[ "${MEMORY_IMPORT_ONLY_HISTORY:-0}" == "1" ]] && import_args+=(--only-history) + echo "memory_transfer: importing ${MEMORY_IMPORT_FILE}" + su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m src.memory_transfer import "$@"' \ + -- sh "$OMEGACLAW_DIR" "${import_args[@]}" \ + || { echo "Memory import failed. Aborting startup." >&2; exit 1; } + echo "memory_transfer: import complete" +fi + # Scrub environment: only allowlisted vars survive. SAFE_VARS="HOME USER PATH HOSTNAME TERM LANG LC_ALL \ PYTHONDONTWRITEBYTECODE PYTHONUNBUFFERED \ HF_HOME SENTENCE_TRANSFORMERS_HOME HF_HUB_OFFLINE TRANSFORMERS_OFFLINE \ + EMBEDDING_PROVIDER \ + OMEGACLAW_memoryExportEnabled \ OMEGACLAW_DIR MEMORY_DIR TEST_SERVER_IP" env_args="" diff --git a/profile/policy.yaml b/profile/policy.yaml index b479db42..01e5227d 100644 --- a/profile/policy.yaml +++ b/profile/policy.yaml @@ -14,6 +14,7 @@ filesystem_policy: - /PeTTa read_write: - /PeTTa/repos/OmegaClaw-Core/memory + - /memory-transfer - /tmp - /dev/null - /opt/huggingface diff --git a/scripts/omegaclaw b/scripts/omegaclaw index ff0ef4ac..5751c8b2 100755 --- a/scripts/omegaclaw +++ b/scripts/omegaclaw @@ -14,6 +14,7 @@ chmod 0600 "$tmp_py_file" cat >"$tmp_py_file" <<'PY' import getpass import json +import os import secrets import string import shlex @@ -282,6 +283,42 @@ def _choose_import_kb(): print("Please enter y or n.") +def _choose_memory_transfer(): + while True: + answer = input("Set up a directory for memory transfer archives? [y/N]: ").strip().lower() + if answer in ("", "n", "no"): + return "", "0" + if answer == "q": + sys.exit(1) + if answer not in ("y", "yes"): + print("Please enter y or n.") + continue + + value = input("Host directory for memory archives: ").strip() + if not value: + print("A directory is required.", file=sys.stderr) + continue + path = os.path.abspath(os.path.expanduser(value)) + try: + os.makedirs(path, exist_ok=True) + except OSError as exc: + print(f"Could not create {path}: {exc}", file=sys.stderr) + continue + if not os.access(path, os.W_OK): + print(f"Directory is not writable: {path}", file=sys.stderr) + continue + + while True: + enable = input("Enable authenticated memory export for this instance? [y/N]: ").strip().lower() + if enable in ("", "n", "no"): + return path, "0" + if enable in ("y", "yes"): + return path, "1" + if enable == "q": + sys.exit(1) + print("Please enter y or n.") + + def _prompt_llm_token(): while True: token = getpass.getpass("Please paste your LLM token and press ENTER or 'q' to exit: ").strip() @@ -309,6 +346,7 @@ def config_run_omegaclaw(config_output_path): channel_config = _choose_channel() provider, embeddingprovider, api_token_var, model, openaiapi_url, token = _choose_provider() import_kb_on_start = _choose_import_kb() + memory_transfer_dir, memory_export_enabled = _choose_memory_transfer() with open(config_output_path, "w", encoding="utf-8") as f: _write_kv(f, "api_token_var", api_token_var) @@ -319,6 +357,8 @@ def config_run_omegaclaw(config_output_path): _write_kv(f, "model", model) _write_kv(f, "openaiapi_url", openaiapi_url) _write_kv(f, "IMPORT_KB_ON_START", import_kb_on_start) + _write_kv(f, "memory_transfer_dir", memory_transfer_dir) + _write_kv(f, "memory_export_enabled", memory_export_enabled) for key, value in channel_config.items(): _write_kv(f, key, value) @@ -497,6 +537,22 @@ help() { echo -e "\t-l set Python logging config file" echo -e "\t--version, -v show the OmegaClaw version" echo -e "\t--help, -h show this help" + echo + echo -e "Memory portability options:" + echo -e "\t--memory-transfer-dir mount host directory for memory export/import archives" + echo -e "\t--enable-memory-export enable authenticated memory export" + echo -e "\t--memory-import restore archive from transfer directory before startup" + echo -e "\t--memory-mode overwrite|append import mode (default: overwrite)" + echo -e "\t--only-history restore history only" + echo -e "\t--no-history skip history during import" + echo -e "\t--no-vector skip long-term memory during import" +} + +require_option_value() { + if [[ "$#" -lt 2 || -z "${2}" ]]; then + echo "${1} requires a value" >&2 + return 1 + fi } options() { @@ -515,6 +571,13 @@ options() { OMEGACLAW_OPENCLAW_TOKEN="${OMEGACLAW_OPENCLAW_TOKEN:-}" commchannel=irc log_config_path="" + memory_transfer_dir="" + memory_import_file="" + memory_import_mode="overwrite" + memory_import_no_history=0 + memory_import_no_vector=0 + memory_import_only_history=0 + memory_export_enabled=0 if [[ "$#" -eq 0 ]]; then help @@ -524,18 +587,61 @@ options() { command=${1} shift 1 - while getopts s:t:p:u:g:c:d:l:m: flag - do - case "${flag}" in - s) OMEGACLAW_AUTH_SECRET=${OPTARG};; - t) commchannel=${OPTARG};; - p) provider=${OPTARG};; - u) openaiapi_url=${OPTARG};; - g) openclaw_url=${OPTARG};; - c) IRC_channel=${OPTARG};; - d) image=${OPTARG};; - l) log_config_path=${OPTARG};; - m) model=${OPTARG};; + while [[ "$#" -gt 0 ]]; do + case "${1}" in + -s?*) OMEGACLAW_AUTH_SECRET=${1:2}; shift; continue;; + -t?*) commchannel=${1:2}; shift; continue;; + -p?*) provider=${1:2}; shift; continue;; + -u?*) openaiapi_url=${1:2}; shift; continue;; + -g?*) openclaw_url=${1:2}; shift; continue;; + -c?*) IRC_channel=${1:2}; shift; continue;; + -d?*) image=${1:2}; shift; continue;; + -l?*) log_config_path=${1:2}; shift; continue;; + -m?*) model=${1:2}; shift; continue;; + -s|-t|-p|-u|-g|-c|-d|-l|-m|--memory-transfer-dir|--memory-import|--memory-mode) + require_option_value "${1}" "${2:-}" || return 1 + ;; + esac + case "${1}" in + -s) OMEGACLAW_AUTH_SECRET=${2}; shift 2;; + -t) commchannel=${2}; shift 2;; + -p) provider=${2}; shift 2;; + -u) openaiapi_url=${2}; shift 2;; + -g) openclaw_url=${2}; shift 2;; + -c) IRC_channel=${2}; shift 2;; + -d) image=${2}; shift 2;; + -l) log_config_path=${2}; shift 2;; + -m) model=${2}; shift 2;; + --memory-transfer-dir) + memory_transfer_dir="${2}" + shift 2 + ;; + --enable-memory-export) + memory_export_enabled=1 + shift + ;; + --memory-import) + memory_import_file="${2}" + shift 2 + ;; + --memory-mode) + memory_import_mode="${2}" + shift 2 + ;; + --no-history) + memory_import_no_history=1 + shift + ;; + --no-vector) + memory_import_no_vector=1 + shift + ;; + --only-history) + memory_import_only_history=1 + shift + ;; + --version|-v) version; return 0;; + --help|-h) help; return 0;; *) help return 1 ;; @@ -583,6 +689,59 @@ options() { return 1 fi fi + + if [[ -n "${memory_transfer_dir}" ]]; then + if [[ "${memory_transfer_dir}" != /* ]]; then + echo "--memory-transfer-dir must be an absolute path: ${memory_transfer_dir}" >&2 + return 1 + fi + if [[ ! -d "${memory_transfer_dir}" ]]; then + echo "--memory-transfer-dir does not exist: ${memory_transfer_dir}" >&2 + return 1 + fi + if [[ ! -w "${memory_transfer_dir}" ]]; then + echo "--memory-transfer-dir is not writable: ${memory_transfer_dir}" >&2 + return 1 + fi + fi + + if [[ -n "${memory_import_file}" ]]; then + if [[ ! "${memory_import_file}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.tar\.gz$ ]]; then + echo "--memory-import must be a plain filename, not a path: ${memory_import_file}" >&2 + return 1 + fi + if [[ -z "${memory_transfer_dir}" ]]; then + echo "--memory-import requires --memory-transfer-dir to be set" >&2 + return 1 + fi + if [[ ! -f "${memory_transfer_dir}/${memory_import_file}" ]]; then + echo "--memory-import archive not found: ${memory_transfer_dir}/${memory_import_file}" >&2 + return 1 + fi + fi + + if [[ "${memory_export_enabled}" == "1" && -z "${memory_transfer_dir}" ]]; then + echo "--enable-memory-export requires --memory-transfer-dir" >&2 + return 1 + fi + + if [[ "${memory_import_no_history}" == "1" && + ( "${memory_import_no_vector}" == "1" || "${memory_import_only_history}" == "1" ) ]]; then + echo "--no-history cannot be combined with --no-vector or --only-history" >&2 + return 1 + fi + if [[ -z "${memory_import_file}" && + ( "${memory_import_no_history}" == "1" || + "${memory_import_no_vector}" == "1" || + "${memory_import_only_history}" == "1" ) ]]; then + echo "Import component flags require --memory-import" >&2 + return 1 + fi + + if [[ "${memory_import_mode}" != "overwrite" && "${memory_import_mode}" != "append" ]]; then + echo "--memory-mode must be overwrite or append" >&2 + return 1 + fi } start() { @@ -591,6 +750,10 @@ start() { container_log_config_path="" log_config_volume=() + memory_transfer_volume=() + memory_import_env=() + memory_transfer_env=() + memory_export_env=() if [ -n "${log_config_path}" ]; then if [ ! -f "${log_config_path}" ]; then @@ -603,6 +766,31 @@ start() { log_config_volume=(--volume "${log_config_abs}:${container_log_config_path}:ro") fi + if [ -n "${memory_transfer_dir}" ]; then + memory_transfer_volume=(--volume "${memory_transfer_dir}:/memory-transfer") + memory_transfer_env=(-e MEMORY_TRANSFER_MOUNTED=1) + fi + + if [[ "${memory_export_enabled}" == "1" ]]; then + memory_export_env=(-e OMEGACLAW_memoryExportEnabled=true) + fi + + if [ -n "${memory_import_file}" ]; then + memory_import_env=( + -e "MEMORY_IMPORT_FILE=${memory_import_file}" + -e "MEMORY_IMPORT_MODE=${memory_import_mode}" + ) + if [[ "${memory_import_no_history}" == "1" ]]; then + memory_import_env+=(-e MEMORY_IMPORT_NO_HISTORY=1) + fi + if [[ "${memory_import_no_vector}" == "1" ]]; then + memory_import_env+=(-e MEMORY_IMPORT_NO_VECTOR=1) + fi + if [[ "${memory_import_only_history}" == "1" ]]; then + memory_import_env+=(-e MEMORY_IMPORT_ONLY_HISTORY=1) + fi + fi + docker_cmd=( docker run -d -it --name omegaclaw @@ -614,12 +802,16 @@ start() { --tmpfs /run:size=16m,mode=755 --volume omegaclaw-memory:/PeTTa/repos/OmegaClaw-Core/memory ${log_config_volume[@]+"${log_config_volume[@]}"} + ${memory_transfer_volume[@]+"${memory_transfer_volume[@]}"} -e "${api_token_var}"="${api_token}" -e "TG_BOT_TOKEN=${TG_BOT_TOKEN:-}" -e "SL_BOT_TOKEN=${SL_BOT_TOKEN:-}" -e "OMEGACLAW_OPENCLAW_TOKEN=${OMEGACLAW_OPENCLAW_TOKEN:-}" -e OMEGACLAW_AUTH_SECRET="$OMEGACLAW_AUTH_SECRET" -e IMPORT_KB_ON_START="${IMPORT_KB_ON_START:-0}" + ${memory_transfer_env[@]+"${memory_transfer_env[@]}"} + ${memory_export_env[@]+"${memory_export_env[@]}"} + ${memory_import_env[@]+"${memory_import_env[@]}"} "$image" "commchannel=${commchannel}" "provider=${provider}" From ba46ca7fdbbe011b1f607b32d9a8e91ee6793ecf Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Mon, 17 Aug 2026 19:59:19 +0300 Subject: [PATCH 04/34] feat(channels): add authenticated memory export controls --- channels/irc.py | 27 ++++- channels/mattermost.py | 37 +++++- channels/memory_export_handler.py | 171 ++++++++++++++++++++++++++++ channels/slack.py | 22 +++- channels/telegram.py | 24 +++- channels/wschat.py | 4 + tests/test_memory_export_handler.py | 75 ++++++++++++ 7 files changed, 356 insertions(+), 4 deletions(-) create mode 100644 channels/memory_export_handler.py create mode 100644 tests/test_memory_export_handler.py diff --git a/channels/irc.py b/channels/irc.py index 82a245ba..b3512304 100644 --- a/channels/irc.py +++ b/channels/irc.py @@ -9,6 +9,7 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key +from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -87,6 +88,20 @@ def _deliver_outbound(chunk): _send(f"PRIVMSG {_channel} :{chunk}") +def _send_export_reply(nick, text): + """Send export-control output privately to the authenticated requester.""" + target = _normalize_nick(nick) + if not target or any(char.isspace() for char in target): + raise ValueError("Invalid IRC nickname for export response") + for line in textwrap.wrap( + str(text).replace("\r", "").replace("\n", " "), + width=400, + break_long_words=True, + break_on_hyphens=False, + ): + _send(f"PRIVMSG {target} :{line}") + + def _flush_outbox(): try: _outbox.flush(_deliver_outbound, _ready_to_send) @@ -155,7 +170,17 @@ def _irc_session(channel, server, port, nick): msg = trailing.split(" :", 1)[1] state = _is_allowed_message(sender_nick, msg) if state == "allow": - _set_last(f"{sender_nick}: {msg}") + if is_export_command(msg): + owner_key = f"irc:{_normalize_nick(sender_nick)}" + reply = handle_export_command( + msg, + owner_key, + lambda message, target=sender_nick: _send_export_reply(target, message), + ) + if reply is not None: + _send_export_reply(sender_nick, reply) + else: + _set_last(f"{sender_nick}: {msg}") elif state == "auth_bound": send_message(f"Authentication successful for {sender_nick}.") except Exception as e: diff --git a/channels/mattermost.py b/channels/mattermost.py index 51a42f91..6aaba97a 100644 --- a/channels/mattermost.py +++ b/channels/mattermost.py @@ -10,6 +10,7 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key +from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -28,6 +29,7 @@ MM_URL = "https://chat.singularitynet.io" CHANNEL_ID = "8fjrmabjx7gupy7e5kjznpt5qh" #NOT AN ID JUST NAME: "omegaclaw"x BOT_TOKEN = "" +BOT_USER_ID = "" def _get_bot_user_id(): global headers @@ -108,6 +110,29 @@ def _deliver_outbound(text): response.raise_for_status() +def _send_export_reply(user_id, text): + """Deliver export-control output through the requester's Mattermost DM.""" + if not BOT_USER_ID: + raise RuntimeError("Mattermost bot identity is not initialized") + direct = requests.post( + f"{MM_URL}/api/v4/channels/direct", + headers=_headers, + json=[BOT_USER_ID, user_id], + timeout=15, + ) + direct.raise_for_status() + direct_channel_id = str(direct.json().get("id", "")).strip() + if not direct_channel_id: + raise RuntimeError("Mattermost did not return a direct-message channel") + response = requests.post( + f"{MM_URL}/api/v4/posts", + headers=_headers, + json={"channel_id": direct_channel_id, "message": str(text)}, + timeout=15, + ) + response.raise_for_status() + + def _flush_outbox(): try: _outbox.flush(_deliver_outbound, _ready_to_send) @@ -157,7 +182,17 @@ def _ws_session(): state = _is_allowed_message(user_id, message) if state == "allow": name = _get_display_name(user_id) - _set_last(f"{name}: {message}") + if is_export_command(message): + owner_key = f"mattermost:{CHANNEL_ID}:{user_id}" + reply = handle_export_command( + message, + owner_key, + lambda text, target=user_id: _send_export_reply(target, text), + ) + if reply is not None: + _send_export_reply(user_id, reply) + else: + _set_last(f"{name}: {message}") elif state == "auth_bound": name = _get_display_name(user_id) send_message(f"Authentication successful for {name}.") diff --git a/channels/memory_export_handler.py b/channels/memory_export_handler.py new file mode 100644 index 00000000..d3492a29 --- /dev/null +++ b/channels/memory_export_handler.py @@ -0,0 +1,171 @@ +"""Shared authenticated /memory-export command handling.""" + +import secrets +import threading +import time + +import auth +from src.logger import get_logger +from src.memory_transfer import get_export_status, is_export_enabled, start_export_job + +logger = get_logger(__name__) + +_TOKEN_TTL_SECONDS = 60 + +_token_lock = threading.Lock() +_pending_requests: dict[str, tuple[str, str, float]] = {} +_job_owners: dict[str, str] = {} + +_VALID_COMPONENTS = ("history", "ltm", "both") + +def is_export_command(text: str) -> bool: + """Return whether text is reserved for the memory-export control plane.""" + command = text.strip().split(None, 1) + if not command: + return False + name = command[0].lower() + return name == "/memory-export" or name.startswith("/memory-export@") + +def _command_arguments(text: str) -> str: + """Return arguments after the plain or Telegram-qualified command name.""" + parts = text.strip().split(None, 1) + return parts[1].strip() if len(parts) == 2 else "" + +def _issue_token(owner_key: str, component: str) -> str: + """Generate and store an owner-scoped confirmation token.""" + token = secrets.token_hex(8) + _pending_requests[owner_key] = ( + token, + component, + time.monotonic() + _TOKEN_TTL_SECONDS, + ) + return token + +def handle_export_command( + text: str, + owner_key: str = "default-owner", + deliver_completion=lambda _message: None, +) -> str | None: + """ + Parse and handle a /memory-export command from the authenticated owner. + + Returns a reply string to send back to the owner, or None when policy + disables export. Callers must use is_export_command() to consume all + reserved control commands before they can reach the LLM. + + text: the raw message text, e.g. "/memory-export history" + """ + stripped = text.strip() + + if not is_export_command(stripped): + return None + + if not auth.is_auth_enabled(): + return None + if not is_export_enabled(): + return None + + rest = _command_arguments(stripped) + parts = rest.split(None, 1) + sub = parts[0].lower() if parts else "" + arg = parts[1].strip() if len(parts) > 1 else "" + + if sub in _VALID_COMPONENTS: + return _handle_request(owner_key, sub) + + if sub == "confirm": + return _handle_confirm(owner_key, arg, deliver_completion) + + if sub == "status": + return _handle_status(owner_key, arg) + + return ( + "Unknown /memory-export command. " + "Use: /memory-export history|ltm|both or " + "/memory-export confirm or " + "/memory-export status " + ) + +def _handle_request(owner_key: str, component: str) -> str: + with _token_lock: + token = _issue_token(owner_key, component) + logger.info(f"memory_export_handler: issued confirmation token for component={component}") + return ( + f"Export requested for: {component}\n" + f"Confirm within {_TOKEN_TTL_SECONDS}s:\n" + f"/memory-export confirm {token}" + ) + +def _handle_confirm(owner_key: str, token: str, deliver_completion) -> str: + if not token: + return "Usage: /memory-export confirm " + + with _token_lock: + pending = _pending_requests.get(owner_key) + if pending is None: + return "No pending export request. Start with /memory-export history|ltm|both" + expected_token, component, expires_at = pending + if time.monotonic() > expires_at: + del _pending_requests[owner_key] + return "Confirmation token expired. Start again with /memory-export history|ltm|both" + if not secrets.compare_digest(expected_token, token): + return "Invalid token." + del _pending_requests[owner_key] + + try: + job_id = start_export_job( + component, + lambda completed_job_id, status: deliver_completion( + _format_completion(completed_job_id, status) + ), + ) + except Exception as exc: + logger.exception(f"memory_export_handler: failed to start export job: {exc}") + return f"Export failed to start: {exc}" + + logger.info(f"memory_export_handler: export job {job_id} started (component={component})") + with _token_lock: + _job_owners[job_id] = owner_key + return ( + f"Export started. Job ID: {job_id}\n" + f"Check progress: /memory-export status {job_id}" + ) + +def _handle_status(owner_key: str, job_id: str) -> str: + if not job_id: + return "Usage: /memory-export status " + + with _token_lock: + if _job_owners.get(job_id) != owner_key: + return f"Export {job_id}: unknown job ID" + + status = get_export_status(job_id) + state = status.get("status", "unknown") + + if state == "running": + return f"Export {job_id}: running…" + + if state == "done": + return ( + f"Export {job_id}: done\n" + f"File: {status.get('filename')}\n" + f"Size: {status.get('size')} bytes\n" + f"SHA-256: {status.get('checksum')}\n" + f"Records: {status.get('record_count')}" + ) + + if state == "failed": + return f"Export {job_id}: failed — {status.get('error')}" + + return f"Export {job_id}: unknown job ID" + +def _format_completion(job_id: str, status: dict) -> str: + if status.get("status") == "done": + return ( + f"Export {job_id}: done\n" + f"File: {status.get('filename')}\n" + f"Size: {status.get('size')} bytes\n" + f"SHA-256: {status.get('checksum')}\n" + f"Records: {status.get('record_count')}" + ) + return f"Export {job_id}: failed — {status.get('error')}" diff --git a/channels/slack.py b/channels/slack.py index a631ec78..950eccad 100644 --- a/channels/slack.py +++ b/channels/slack.py @@ -11,6 +11,7 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key +from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -361,7 +362,17 @@ def _poll_channel(channel_id): state = _is_allowed_message(channel_id, user_id, text) display_name = _get_display_name(user_id) if state == "allow": - _set_last(f"<@{user_id}> ({display_name}): {text}") + if is_export_command(text): + owner_key = f"slack:{channel_id}:{user_id}" + reply = handle_export_command( + text, + owner_key, + lambda message, target=user_id: _send_export_reply(target, message), + ) + if reply is not None: + _send_export_reply(user_id, reply) + else: + _set_last(f"{display_name}: {text}") elif state == "auth_bound": send_message(f"Authentication successful for {display_name}.") @@ -387,6 +398,15 @@ def _deliver_outbound(chunk): ) +def _send_export_reply(user_id, text): + """Deliver export-control output through the requester's Slack DM.""" + payload = _api_call("conversations.open", {"users": user_id}, timeout=15) + channel_id = str((payload.get("channel") or {}).get("id", "")).strip() + if not channel_id: + raise RuntimeError("Slack did not return a direct-message channel") + _api_call("chat.postMessage", {"channel": channel_id, "text": str(text)}, timeout=15) + + def _flush_outbox(): try: _outbox.flush(_deliver_outbound, _ready_to_send) diff --git a/channels/telegram.py b/channels/telegram.py index 331da7ff..129a553e 100644 --- a/channels/telegram.py +++ b/channels/telegram.py @@ -9,6 +9,7 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key +from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -163,6 +164,16 @@ def _deliver_outbound(chunk): ) +def _send_export_reply(user_id, text): + """Send private export-control output to the authenticated requester.""" + _api_call( + "sendMessage", + {"chat_id": user_id, "text": str(text)}, + timeout=15, + use_post=True, + ) + + def _flush_outbox(): global _connected try: @@ -210,8 +221,19 @@ def _poll_loop(): state = _is_allowed_message(chat_id, user_id, text) display_name = _display_name(user, chat) + export_command = is_export_command(text) if state == "allow": - _set_last(f"{display_name}: {text}") + if export_command: + owner_key = f"telegram:{chat_id}:{user_id}" + reply = handle_export_command( + text, + owner_key, + lambda message, target=user_id: _send_export_reply(target, message), + ) + if reply is not None: + _send_export_reply(user_id, reply) + else: + _set_last(f"{display_name}: {text}") elif state == "auth_bound": send_message(f"Authentication successful for {display_name}.") _flush_outbox() diff --git a/channels/wschat.py b/channels/wschat.py index d188a58a..968b237d 100644 --- a/channels/wschat.py +++ b/channels/wschat.py @@ -64,6 +64,7 @@ from pathlib import Path import sys from config import config_get_by_key +from memory_export_handler import is_export_command _REPO_ROOT = Path(__file__).resolve().parents[1] if str(_REPO_ROOT) not in sys.path: @@ -194,6 +195,9 @@ def _handle_frame(raw_message): if not isinstance(seq, int) or not isinstance(text, str): logger.warning(f"Ignoring malformed user_message frame: {frame!r}") return + if is_export_command(text): + logger.info("Ignoring unavailable memory-export command on WebSocket channel") + return _enqueue_user_message(seq, text) return diff --git a/tests/test_memory_export_handler.py b/tests/test_memory_export_handler.py new file mode 100644 index 00000000..a3273cc1 --- /dev/null +++ b/tests/test_memory_export_handler.py @@ -0,0 +1,75 @@ +import importlib.util +import sys +import time +import types +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def handler(monkeypatch): + auth = types.ModuleType("auth") + auth.is_auth_enabled = lambda: True + monkeypatch.setitem(sys.modules, "auth", auth) + logger = types.ModuleType("src.logger") + logger.get_logger = lambda name: __import__("logging").getLogger(name) + monkeypatch.setitem(sys.modules, "src.logger", logger) + transfer = types.ModuleType("src.memory_transfer") + transfer.is_export_enabled = lambda: True + transfer.start_export_job = lambda component, on_complete=None: "job-1" + transfer.get_export_status = lambda job_id: {"status": "unknown"} + monkeypatch.setitem(sys.modules, "src.memory_transfer", transfer) + spec = importlib.util.spec_from_file_location( + "memory_export_handler_under_test", REPO_ROOT / "channels" / "memory_export_handler.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_export_command_requires_auth_and_policy(handler): + handler.auth.is_auth_enabled = lambda: False + assert handler.handle_export_command("/memory-export both") is None + handler.auth.is_auth_enabled = lambda: True + handler.is_export_enabled = lambda: False + assert handler.handle_export_command("/memory-export both") is None + + +def test_confirmation_starts_only_the_requested_export(handler): + started = [] + handler.start_export_job = lambda component, on_complete: started.append(component) or "job-1" + token = handler.handle_export_command("/memory-export ltm").split()[-1] + + assert "Invalid token" in handler.handle_export_command("/memory-export confirm wrong") + assert "job-1" in handler.handle_export_command(f"/memory-export confirm {token}") + assert started == ["ltm"] + + +def test_expired_and_other_owner_tokens_cannot_start_export(handler): + token = handler.handle_export_command("/memory-export history", "owner-a").split()[-1] + assert "No pending export" in handler.handle_export_command( + f"/memory-export confirm {token}", "owner-b" + ) + token_state = handler._pending_requests["owner-a"] + handler._pending_requests["owner-a"] = (*token_state[:2], time.monotonic() - 1) + assert "expired" in handler.handle_export_command( + f"/memory-export confirm {token}", "owner-a" + ).lower() + + +def test_completion_and_status_are_limited_to_requesting_owner(handler): + delivered = [] + + def start_job(component, callback): + callback("job-1", {"status": "done", "filename": "memory.tar.gz"}) + return "job-1" + + handler.start_export_job = start_job + token = handler.handle_export_command("/memory-export both", "owner-a", delivered.append).split()[-1] + handler.handle_export_command(f"/memory-export confirm {token}", "owner-a", delivered.append) + + assert "memory.tar.gz" in delivered[0] + assert "unknown job ID" in handler.handle_export_command("/memory-export status job-1", "owner-b") From 813bfcf55fc0b7f70bc77959a1f83a65257d9376 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Mon, 17 Aug 2026 19:59:35 +0300 Subject: [PATCH 05/34] docs(memory): document portability workflow --- README.md | 13 +++++ docs/README.md | 2 + docs/reference-memory-portability.md | 76 ++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 docs/reference-memory-portability.md diff --git a/README.md b/README.md index 44ba26f0..d11eb869 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,19 @@ To reset OmegaClaw's memory: docker volume rm omegaclaw-memory ``` +### Memory portability + +Memory export is disabled by default. See the [memory portability reference](./docs/reference-memory-portability.md) +for setup, export controls, archive contents, and import modes. To restore an +archive while upgrading to a tagged image, use the same transfer directory: + +```sh +scripts/omegaclaw start -d singularitynet/omegaclaw: -p OpenAI -t telegram \ + --memory-transfer-dir "$HOME/omegaclaw-transfers" \ + --memory-import omegaclaw-memory-.tar.gz \ + --memory-mode overwrite +``` + --- ## Usage diff --git a/docs/README.md b/docs/README.md index 2a6c8a21..0afd95fe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,6 +57,8 @@ User-facing MeTTa skills the agent invokes. Each page follows the template **Sig - [reference-configuration.md](./reference-configuration.md) — `configure` form and all runtime parameters - [reference-channels.md](./reference-channels.md) — IRC, Telegram, Slack, Mattermost, WebSocket, and websearch adapters plus the channel contract - [reference-python-bridges.md](./reference-python-bridges.md) — `lib_llm_ext.py`, `src/helper.py`, `src/skills.pl` +- [reference-memory-portability.md](./reference-memory-portability.md) — Operator backup, restore, and archive-transfer workflow +- [reference-python-bridges.md](./reference-python-bridges.md) — `lib_llm_ext.py`, `src/agentverse.py`, `src/helper.py`, `src/skills.pl` ### Internals diff --git a/docs/reference-memory-portability.md b/docs/reference-memory-portability.md new file mode 100644 index 00000000..c46d39a8 --- /dev/null +++ b/docs/reference-memory-portability.md @@ -0,0 +1,76 @@ +# Reference - Memory Portability + +Memory portability lets an operator export persistent user memory from one +deployment and restore it before another agent starts. It is an operator +workflow, not an LLM skill. + +## Setup + +Choose an absolute host directory for archives. The launcher mounts it at the +fixed container path `/memory-transfer`; the agent never accepts arbitrary +runtime export paths. + +```sh +scripts/omegaclaw start -p OpenAI -t telegram \ + --memory-transfer-dir "$HOME/omegaclaw-transfers" \ + --enable-memory-export +``` + +`--enable-memory-export` is required because export is disabled by default. +The transfer directory must be writable by the container's agent user. + +## Export + +In a private, authenticated supported chat, request one component and confirm +the returned short-lived token: + +```text +/memory-export history +/memory-export ltm +/memory-export both +/memory-export confirm +/memory-export status +``` + +The export runs in the background. Completion is delivered only to the owner +who started it and includes the filename, record count, size, and SHA-256. + +Archives contain selected persistent user memory only: + +```text +manifest.json +history/history.metta +vector/collections.json +vector/records.jsonl +``` + +History is the conversation trace. LTM is logical user-memory records from +ChromaDB. Prompts, credentials, logs, skills, and other operational state are +not exported. SHA-256 detects corruption, not archive authorship. + +## Import + +Import is an administrative startup operation. The archive argument is a plain +filename in the chosen transfer directory: + +```sh +scripts/omegaclaw start -d singularitynet/omegaclaw: -p OpenAI -t telegram \ + --memory-transfer-dir "$HOME/omegaclaw-transfers" \ + --memory-import omegaclaw-memory-.tar.gz \ + --memory-mode overwrite +``` + +`overwrite` replaces the selected components after validation and rollback +preparation. `append` preserves existing history and adds imported LTM records +under new IDs. Select components with `--only-history`, `--no-history`, or +`--no-vector`. + +The importer validates archive paths, checksums, manifest metadata, record +counts, and embedding compatibility before changing live memory. It runs before +the agent loop starts. A receipt prevents a completed archive import from +running again on container restart. + +## Limits + +Memory export commands are not supported on the WebSocket chat channel. +Archives are private operator data; keep the host transfer directory protected. From 2a62fcda67a6d08c8a9580e01367f38d7ca4fe65 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Wed, 19 Aug 2026 22:10:21 +0300 Subject: [PATCH 06/34] build(docker): install memory portability package --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d43eb119..95de4603 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,7 +58,6 @@ RUN python3 -m pip install --no-cache-dir --break-system-packages \ --extra-index-url https://pypi.org/simple/ \ torch==2.12.1 \ && python3 -m pip install --no-cache-dir --break-system-packages -r /tmp/requirements.txt - # Pre-download the sentence-transformers model so runtime does not need network access. RUN mkdir -p "${HF_HOME}" "${SENTENCE_TRANSFORMERS_HOME}" \ && python3 - < Date: Wed, 19 Aug 2026 22:10:37 +0300 Subject: [PATCH 07/34] refactor(memory): use portability package --- entrypoint.sh | 19 +- scripts/omegaclaw | 5 +- src/helper.py | 4 +- src/memory_gateway.py | 50 +- src/memory_layout.py | 19 - src/memory_transfer.py | 878 ---------------------------------- tests/test_memory_gateway.py | 9 - tests/test_memory_layout.py | 19 - tests/test_memory_transfer.py | 273 ----------- 9 files changed, 43 insertions(+), 1233 deletions(-) delete mode 100644 src/memory_layout.py delete mode 100644 src/memory_transfer.py delete mode 100644 tests/test_memory_layout.py delete mode 100644 tests/test_memory_transfer.py diff --git a/entrypoint.sh b/entrypoint.sh index bc92aa7d..de005524 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -40,14 +40,17 @@ if [[ "${IMPORT_KB_ON_START}" == "1" ]]; then su nobody -s /bin/sh -c "${OMEGACLAW_DIR}/scripts/import_knowledge.sh" fi +MEMORY_TRANSFER_DIR="${MEMORY_TRANSFER_DIR:-/memory-transfer}" +export MEMORY_TRANSFER_DIR + # Verify that the agent user can write the mounted transfer directory. if [[ "${MEMORY_TRANSFER_MOUNTED:-0}" == "1" ]]; then - su nobody -s /bin/sh -c 'test -d /memory-transfer && test -w /memory-transfer' \ + su nobody -s /bin/sh -c 'test -d "$1" && test -w "$1"' sh "$MEMORY_TRANSFER_DIR" \ || { echo "Memory transfer directory is not writable by the agent user." >&2; exit 1; } fi # Recover an interrupted import before starting the agent. -su nobody -s /bin/sh -c 'cd "$1" && exec python3 -m src.memory_transfer recover' \ +su nobody -s /bin/sh -c 'cd "$1" && exec python3 -m memory_portability recover' \ sh "$OMEGACLAW_DIR" \ || { echo "Memory import recovery failed. Aborting startup." >&2; exit 1; } @@ -61,15 +64,15 @@ if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then overwrite|append) ;; *) echo "MEMORY_IMPORT_MODE must be overwrite or append" >&2; exit 1 ;; esac - import_args=("/memory-transfer/${MEMORY_IMPORT_FILE}" --mode "${MEMORY_IMPORT_MODE:-overwrite}") + import_args=(--transfer-dir "$MEMORY_TRANSFER_DIR" --filename "${MEMORY_IMPORT_FILE}" --mode "${MEMORY_IMPORT_MODE:-overwrite}") [[ "${MEMORY_IMPORT_NO_HISTORY:-0}" == "1" ]] && import_args+=(--no-history) [[ "${MEMORY_IMPORT_NO_VECTOR:-0}" == "1" ]] && import_args+=(--no-vector) - [[ "${MEMORY_IMPORT_ONLY_HISTORY:-0}" == "1" ]] && import_args+=(--only-history) - echo "memory_transfer: importing ${MEMORY_IMPORT_FILE}" - su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m src.memory_transfer import "$@"' \ + [[ "${MEMORY_IMPORT_ONLY_HISTORY:-0}" == "1" ]] && import_args+=(--no-vector) + echo "memory_portability: importing ${MEMORY_IMPORT_FILE}" + su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m memory_portability import "$@"' \ -- sh "$OMEGACLAW_DIR" "${import_args[@]}" \ || { echo "Memory import failed. Aborting startup." >&2; exit 1; } - echo "memory_transfer: import complete" + echo "memory_portability: import complete" fi # Scrub environment: only allowlisted vars survive. @@ -78,7 +81,7 @@ SAFE_VARS="HOME USER PATH HOSTNAME TERM LANG LC_ALL \ HF_HOME SENTENCE_TRANSFORMERS_HOME HF_HUB_OFFLINE TRANSFORMERS_OFFLINE \ EMBEDDING_PROVIDER \ OMEGACLAW_memoryExportEnabled \ - OMEGACLAW_DIR MEMORY_DIR TEST_SERVER_IP" + OMEGACLAW_DIR MEMORY_DIR MEMORY_TRANSFER_DIR TEST_SERVER_IP" env_args="" for var in $SAFE_VARS; do diff --git a/scripts/omegaclaw b/scripts/omegaclaw index 5751c8b2..570b538a 100755 --- a/scripts/omegaclaw +++ b/scripts/omegaclaw @@ -768,7 +768,10 @@ start() { if [ -n "${memory_transfer_dir}" ]; then memory_transfer_volume=(--volume "${memory_transfer_dir}:/memory-transfer") - memory_transfer_env=(-e MEMORY_TRANSFER_MOUNTED=1) + memory_transfer_env=( + -e MEMORY_TRANSFER_MOUNTED=1 + -e MEMORY_TRANSFER_DIR=/memory-transfer + ) fi if [[ "${memory_export_enabled}" == "1" ]]; then diff --git a/src/helper.py b/src/helper.py index 1f90a1a8..19742037 100644 --- a/src/helper.py +++ b/src/helper.py @@ -9,10 +9,10 @@ try: from src.logger import get_logger - from src.memory_layout import history_path + from src.memory_gateway import history_path except ModuleNotFoundError: # running this file directly as a script from logger import get_logger - from memory_layout import history_path + from memory_gateway import history_path logger = get_logger(__name__) diff --git a/src/memory_gateway.py b/src/memory_gateway.py index fff0a8c4..486ef66b 100644 --- a/src/memory_gateway.py +++ b/src/memory_gateway.py @@ -1,53 +1,58 @@ """Thread-safe access to persistent history and vector memory.""" +import os import threading import uuid +from pathlib import Path + import chromadb -from src.memory_layout import history_path, chroma_db_path from src.logger import get_logger logger = get_logger(__name__) _write_lock = threading.Lock() - _RECORD_KIND = "user_memory" - _client: chromadb.ClientAPI | None = None _collection = None +_REPO_ROOT = Path(__file__).parent.parent.resolve() + +def memory_dir_path() -> Path: + return Path(os.environ.get("MEMORY_DIR", _REPO_ROOT / "memory")).resolve() + +def history_path() -> Path: + return memory_dir_path() / "history.metta" + +def chroma_db_path() -> Path: + return memory_dir_path() / "chroma_db" def _get_collection(): - """Lazy-initialise the ChromaDB client and collection.""" global _client, _collection if _collection is None: db_path = str(chroma_db_path()) logger.info(f"memory_gateway: opening ChromaDB at {db_path}") _client = chromadb.PersistentClient(path=db_path) _collection = _client.get_or_create_collection( - name="memories", - embedding_function=None, + name="memories", embedding_function=None ) return _collection def append_history(text: str) -> None: - """Append a history entry and its trailing newline, creating the file.""" path = history_path() path.parent.mkdir(parents=True, exist_ok=True) with _write_lock: - with path.open("a", encoding="utf-8") as f: - f.write(text) - f.write("\n") + with path.open("a", encoding="utf-8") as history: + history.write(text) + history.write("\n") def history_tail(max_chars: int) -> str: - """Return the trailing character window of history, or an empty string.""" path = history_path() if not path.exists(): return "" - with path.open("r", encoding="utf-8", errors="replace") as f: - return f.read()[-max_chars:] + with path.open("r", encoding="utf-8", errors="replace") as history: + return history.read()[-max_chars:] def remember(content: str, embedding: list[float], time: str) -> str: - """Store a user memory record and return its ID.""" item_id = str(uuid.uuid4()) with _write_lock: _get_collection().add( @@ -60,21 +65,18 @@ def remember(content: str, embedding: list[float], time: str) -> str: return item_id def query(query_embedding: list[float], k: int) -> list[list]: - """Return the k most similar records as [time, content].""" - results = query_with_ids(query_embedding, k) - return [[time, content] for _, time, content in results] + return [[time, content] for _, time, content in query_with_ids(query_embedding, k)] def query_with_ids(query_embedding: list[float], k: int) -> list[list]: - """Return the k most similar records as [id, time, content].""" - res = _get_collection().query( + result = _get_collection().query( query_embeddings=[query_embedding], n_results=k, include=["documents", "metadatas", "distances"], ) - ids = res["ids"][0] - docs = res.get("documents", [[]])[0] - metas = res.get("metadatas", [[]])[0] + ids = result["ids"][0] + documents = result.get("documents", [[]])[0] + metadata = result.get("metadatas", [[]])[0] return [ - [ids[i], metas[i].get("time") if metas[i] else None, docs[i]] - for i in range(len(ids)) + [ids[index], metadata[index].get("time") if metadata[index] else None, documents[index]] + for index in range(len(ids)) ] diff --git a/src/memory_layout.py b/src/memory_layout.py deleted file mode 100644 index 28a4819b..00000000 --- a/src/memory_layout.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import pathlib - -_REPO_ROOT = pathlib.Path(__file__).parent.parent.resolve() - -def memory_dir_path() -> pathlib.Path: - """Return MEMORY_DIR or the repository's memory directory.""" - memory_dir = os.environ.get("MEMORY_DIR") - if memory_dir: - return pathlib.Path(memory_dir).resolve() - return _REPO_ROOT / "memory" - -def history_path() -> pathlib.Path: - """Return the history file path.""" - return memory_dir_path() / "history.metta" - -def chroma_db_path() -> pathlib.Path: - """Return the ChromaDB directory path.""" - return memory_dir_path() / "chroma_db" diff --git a/src/memory_transfer.py b/src/memory_transfer.py deleted file mode 100644 index 27556ead..00000000 --- a/src/memory_transfer.py +++ /dev/null @@ -1,878 +0,0 @@ -"""Export and import persistent user memory.""" - -import argparse -import hashlib -import json -import os -import shutil -import tarfile -import tempfile -import threading -import uuid -from collections.abc import Callable -from datetime import datetime, timezone -from pathlib import Path - -import chromadb - -import src.memory_gateway as gateway -from src.memory_layout import chroma_db_path, history_path, memory_dir_path -from src.logger import get_logger -from src.config import config_get_by_key - -logger = get_logger(__name__) - -TRANSFER_DIR = Path("/memory-transfer") -ARCHIVE_FORMAT_VERSION = 1 - -_ALLOWLIST = frozenset([ - "manifest.json", - "history/history.metta", - "vector/collections.json", - "vector/records.jsonl", -]) - -_COMPONENT_FILES = { - "history": {"history/history.metta"}, - "ltm": {"vector/collections.json", "vector/records.jsonl"}, -} - -_MAX_COMPRESSED_BYTES = 500 * 1024 * 1024 # 500 MB -_MAX_EXTRACTED_BYTES = 2 * 1024 * 1024 * 1024 # 2 GB - -_TX_MARKER_NAME = ".import_in_progress" -_RECEIPT_DIR_NAME = ".memory_import_receipts" -_ROLLBACK_STATE_NAME = "state.json" - -_REEMBED_BATCH = 64 -_VECTOR_BATCH = 500 - -def is_export_enabled() -> bool: - """Return whether an administrator enabled memory export.""" - env_val = os.environ.get("OMEGACLAW_memoryExportEnabled") - if env_val is not None: - return env_val.strip().lower() == "true" - - value = config_get_by_key("memoryExportEnabled", False) - if isinstance(value, bool): - return value - if isinstance(value, str): - return value.strip().lower() == "true" - return value is True - -def _sha256(path: Path) -> str: - h = hashlib.sha256() - with path.open("rb") as f: - for chunk in iter(lambda: f.read(65536), b""): - h.update(chunk) - return h.hexdigest() - -def _utc_now() -> str: - return datetime.now(timezone.utc).isoformat() - -def _archive_name() -> str: - timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") - return f"omegaclaw-memory-{timestamp}.tar.gz" - -def _embedding_profile() -> dict: - """Return the active embedding profile used to assess archive compatibility.""" - provider = os.environ.get("EMBEDDING_PROVIDER", "Local") - default_model = "text-embedding-3-large" if provider == "OpenAI" else "intfloat/e5-large-v2" - return { - "provider": provider, - "model": os.environ.get("SENTENCE_TRANSFORMERS_MODEL", default_model), - } - -def _reembed_records(records: list[dict], manifest: dict, - target_dimension: int | None) -> None: - """Regenerate embeddings in bounded batches when archive and runtime differ.""" - source = manifest.get("embedding_info", {}) - active = _embedding_profile() - source_dimension = source.get("vector_dimension") - embeddings_present = all( - record.get("embedding") and len(record["embedding"]) == source_dimension - for record in records - ) - if (source.get("provider") == active["provider"] and - source.get("model") == active["model"] and embeddings_present and - target_dimension in (None, source_dimension)): - return - - logger.warning("memory_transfer: embedding profile mismatch — re-embedding in batches") - from src.rag import local_embed_batch, openai_embed_batch - - embed_fn = openai_embed_batch if active["provider"] == "OpenAI" else local_embed_batch - for i in range(0, len(records), _REEMBED_BATCH): - batch = records[i : i + _REEMBED_BATCH] - embeddings = embed_fn([record["document"] for record in batch]) - for record, embedding in zip(batch, embeddings): - record["embedding"] = embedding - -def _normalise_metadata(metadata: object) -> dict: - """Convert archive metadata into ChromaDB's scalar-only metadata format.""" - if not isinstance(metadata, dict): - raise ValueError("Archive record metadata must be an object") - - normalised = {} - for key, value in metadata.items(): - if not isinstance(key, str) or not key: - raise ValueError("Archive metadata keys must be non-empty strings") - if isinstance(value, (str, int, float, bool)): - normalised[key] = value - elif value is None: - normalised[key] = "null" - elif isinstance(value, (list, dict)): - normalised[key] = json.dumps(value, ensure_ascii=False, sort_keys=True) - else: - normalised[key] = str(value) - return normalised - -def _normalise_record(record: object, line_number: int) -> dict: - """Validate one archive JSONL record and return a Chroma-ready mapping.""" - if not isinstance(record, dict): - raise ValueError(f"Archive record on line {line_number} must be an object") - - record_id = record.get("id") - document = record.get("document") - embedding = record.get("embedding", []) - if not isinstance(record_id, str) or not record_id: - raise ValueError(f"Archive record on line {line_number} has an invalid id") - if not isinstance(document, str): - raise ValueError(f"Archive record {record_id!r} has a non-string document") - if not isinstance(embedding, list) or not all( - isinstance(value, (int, float)) and not isinstance(value, bool) - for value in embedding): - raise ValueError(f"Archive record {record_id!r} has an invalid embedding") - - return { - "id": record_id, - "document": document, - "embedding": embedding, - "metadata": _normalise_metadata(record.get("metadata", {})), - } - -def _record_batches(records_path: Path): - """Yield validated archive records in bounded JSONL batches.""" - batch = [] - with records_path.open("r", encoding="utf-8") as records_file: - for line_number, line in enumerate(records_file, 1): - if not line.strip(): - continue - try: - raw_record = json.loads(line) - except json.JSONDecodeError as exc: - raise ValueError( - f"Invalid JSONL record on line {line_number}: {exc.msg}" - ) from exc - batch.append(_normalise_record(raw_record, line_number)) - if len(batch) == _VECTOR_BATCH: - yield batch - batch = [] - if batch: - yield batch - -def _validate_manifest(manifest: object) -> dict: - if not isinstance(manifest, dict): - raise ValueError("Archive manifest must be an object") - if type(manifest.get("format_version")) is not int: - raise ValueError("Archive manifest has invalid format_version") - for field in ("omegaclaw_version", "chromadb_version", "created_at"): - if not isinstance(manifest.get(field), str) or not manifest[field]: - raise ValueError(f"Archive manifest has invalid {field}") - try: - created_at = datetime.fromisoformat(manifest["created_at"].replace("Z", "+00:00")) - except ValueError as exc: - raise ValueError("Archive manifest has invalid created_at") from exc - if created_at.tzinfo is None or created_at.utcoffset() != timezone.utc.utcoffset(created_at): - raise ValueError("Archive manifest created_at must be UTC") - components = manifest.get("components") - if (not isinstance(components, list) - or any(component not in ("history", "ltm") for component in components) - or len(components) != len(set(components))): - raise ValueError("Archive manifest has invalid components") - if not isinstance(manifest.get("record_count"), int) or isinstance( - manifest["record_count"], bool) or manifest["record_count"] < 0: - raise ValueError("Archive manifest has invalid record_count") - if not isinstance(manifest.get("history_bytes"), int) or isinstance( - manifest["history_bytes"], bool) or manifest["history_bytes"] < 0: - raise ValueError("Archive manifest has invalid history_bytes") - if "ltm" not in components and manifest["record_count"]: - raise ValueError("Archive manifest has records without the ltm component") - if "history" not in components and manifest["history_bytes"]: - raise ValueError("Archive manifest has history bytes without the history component") - checksums = manifest.get("checksums") - if not isinstance(checksums, dict) or not all( - isinstance(name, str) and isinstance(value, str) - and len(value) == 64 and all(char in "0123456789abcdef" for char in value.lower()) - for name, value in checksums.items()): - raise ValueError("Archive manifest has invalid checksums") - expected_files = set().union(*(_COMPONENT_FILES[component] for component in components)) - if set(checksums) != expected_files: - raise ValueError("Archive manifest checksums do not match its components") - if "ltm" in components: - _validate_embedding_info(manifest) - elif manifest.get("embedding_info") != {}: - raise ValueError("Archive manifest has embedding_info without the ltm component") - return manifest - -def _validate_embedding_info(manifest: dict) -> None: - embedding_info = manifest.get("embedding_info") - if (not isinstance(embedding_info, dict) - or not isinstance(embedding_info.get("provider"), str) - or not isinstance(embedding_info.get("model"), str) - or not isinstance(embedding_info.get("vector_dimension"), int) - or isinstance(embedding_info["vector_dimension"], bool) - or embedding_info["vector_dimension"] < 0): - raise ValueError("Archive manifest has invalid embedding_info") - -def _validate_records(records_path: Path, manifest: dict) -> None: - expected_count = manifest["record_count"] - dimension = manifest["embedding_info"]["vector_dimension"] - count = 0 - for records in _record_batches(records_path): - for record in records: - if record["embedding"] and len(record["embedding"]) != dimension: - raise ValueError("Archive record embedding dimension does not match manifest") - count += 1 - if count != expected_count: - raise ValueError("Archive record count does not match manifest") - -def _validate_extracted_archive(staging: Path, manifest: dict) -> None: - """Validate extracted content before import can modify live memory.""" - for member_name, expected in manifest["checksums"].items(): - actual = _sha256(staging / member_name) - if actual != expected: - raise ValueError(f"Checksum mismatch for {member_name!r}") - - components = manifest["components"] - if "history" in components: - history = staging / "history" / "history.metta" - if history.stat().st_size != manifest["history_bytes"]: - raise ValueError("Archive history size does not match manifest") - if "ltm" in components: - collections = json.loads( - (staging / "vector" / "collections.json").read_text(encoding="utf-8") - ) - if (not isinstance(collections, dict) - or collections.get("name") != "memories" - or collections.get("embedding_info") != manifest["embedding_info"]): - raise ValueError("Archive collections metadata does not match manifest") - _validate_records(staging / "vector" / "records.jsonl", manifest) - -def _collection_dimension(collection) -> int | None: - result = collection.get(limit=1, include=["embeddings"]) - embeddings = result.get("embeddings") - return len(embeddings[0]) if embeddings is not None and len(embeddings) else None - -def _staging_dir() -> Path: - """Return same-filesystem staging for atomic archive publication.""" - staging = TRANSFER_DIR / ".staging" - staging.mkdir(parents=True, exist_ok=True) - return staging - -def _is_user_memory_record(metadata: object) -> bool: - """Return whether a collection record belongs to portable user memory.""" - if not isinstance(metadata, dict): - return False - return ( - metadata.get("record_kind") == gateway._RECORD_KIND - or ("record_kind" not in metadata and "type" not in metadata) - ) - -def _export_history(staging: Path) -> int: - """Copy history.metta into staging. Returns byte size.""" - src = history_path() - dst = staging / "history" / "history.metta" - dst.parent.mkdir(parents=True, exist_ok=True) - if src.exists(): - shutil.copy2(src, dst) - return dst.stat().st_size - dst.touch() - return 0 - -def _export_vectors(staging: Path) -> tuple[int, dict]: - """Export marked and legacy user records into staging.""" - vector_dir = staging / "vector" - vector_dir.mkdir(parents=True, exist_ok=True) - records_path = vector_dir / "records.jsonl" - col = gateway._get_collection() - record_count = 0 - dimension = 0 - - with records_path.open("w", encoding="utf-8") as records_file: - for offset in range(0, col.count(), _VECTOR_BATCH): - result = col.get( - limit=_VECTOR_BATCH, - offset=offset, - include=["documents", "metadatas", "embeddings"], - ) - ids = result.get("ids", []) - docs = result.get("documents") or [] - metas = result.get("metadatas") or [] - raw_embeddings = result.get("embeddings") - embeddings = list(raw_embeddings) if raw_embeddings is not None else [] - for index, record_id in enumerate(ids): - metadata = metas[index] if index < len(metas) else {} - if not _is_user_memory_record(metadata): - continue - embedding = embeddings[index] if index < len(embeddings) else [] - embedding = list(embedding) if hasattr(embedding, "__iter__") else embedding - if not dimension and embedding: - dimension = len(embedding) - records_file.write(json.dumps({ - "id": record_id, - "document": docs[index] if index < len(docs) else "", - "metadata": _normalise_metadata(metadata), - "embedding": embedding, - }, ensure_ascii=False) + "\n") - record_count += 1 - - active_profile = _embedding_profile() - embedding_info = { - "provider": active_profile["provider"], - "model": active_profile["model"], - "vector_dimension": dimension, - } - - (vector_dir / "collections.json").write_text( - json.dumps({"name": "memories", "embedding_info": embedding_info}, indent=2), - encoding="utf-8", - ) - - return record_count, embedding_info - -def _build_manifest(staging: Path, components: list[str], - record_count: int, embedding_info: dict) -> None: - """Write manifest.json into staging.""" - from src.helper import omegaclaw_version - - checksums: dict[str, str] = {} - for member in _ALLOWLIST: - p = staging / member - if p.exists(): - checksums[member] = _sha256(p) - - manifest = { - "format_version": ARCHIVE_FORMAT_VERSION, - "omegaclaw_version": omegaclaw_version(), - "chromadb_version": chromadb.__version__, - "components": components, - "embedding_info": embedding_info, - "record_count": record_count, - "history_bytes": (staging / "history" / "history.metta").stat().st_size - if (staging / "history" / "history.metta").exists() else 0, - "created_at": _utc_now(), - "checksums": checksums, - } - (staging / "manifest.json").write_text( - json.dumps(manifest, indent=2), encoding="utf-8" - ) - -def _pack_archive(staging: Path, dest: Path) -> None: - """Pack staging directory into a .tar.gz at dest.""" - with tarfile.open(dest, "w:gz") as tar: - for member in sorted(_ALLOWLIST): - p = staging / member - if p.exists(): - tar.add(p, arcname=member) - -def start_export_job( - component: str, on_complete: Callable[[str, dict], None] | None = None -) -> str: - """Start an asynchronous export and return its job ID.""" - if component not in ("history", "ltm", "both"): - raise ValueError(f"Invalid component: {component!r}. Use history, ltm, or both.") - - job_id = uuid.uuid4().hex - with _jobs_lock: - _jobs[job_id] = {"status": "running"} - threading.Thread( - target=_run_export, args=(job_id, component, on_complete), daemon=True - ).start() - logger.info(f"memory_transfer: export job {job_id} started (component={component})") - return job_id - -def _run_export( - job_id: str, component: str, on_complete: Callable[[str, dict], None] | None -) -> None: - try: - result = export(component) - status = {"status": "done", **result} - except Exception as exc: - logger.exception(f"memory_transfer: export job {job_id} failed: {exc}") - status = {"status": "failed", "error": str(exc)} - - with _jobs_lock: - _jobs[job_id] = status - - if on_complete is not None: - try: - on_complete(job_id, status.copy()) - except Exception as exc: - logger.exception( - f"memory_transfer: completion callback for export job {job_id} failed: {exc}" - ) - -_jobs: dict[str, dict] = {} -_jobs_lock = threading.Lock() - -def get_export_status(job_id: str) -> dict: - with _jobs_lock: - return _jobs.get(job_id, {"status": "unknown"}).copy() - -def export(component: str) -> dict: - """Export selected memory components and publish an archive atomically.""" - include_history = component in ("history", "both") - include_vectors = component in ("ltm", "both") - - archive_name = _archive_name() - work_dir = _staging_dir() / archive_name - work_dir.mkdir(parents=True, exist_ok=True) - staging = work_dir / "staging" - staging.mkdir() - # Landlock permits atomic rename within a directory, but can reject a - # rename between .staging and its parent as a cross-directory operation. - tmp_archive = TRANSFER_DIR / f".{archive_name}.tmp" - - try: - record_count = 0 - embedding_info: dict = {} - components: list[str] = [] - - with gateway._write_lock: - if include_history: - _export_history(staging) - components.append("history") - if include_vectors: - record_count, embedding_info = _export_vectors(staging) - components.append("ltm") - - _build_manifest(staging, components, record_count, embedding_info) - _pack_archive(staging, tmp_archive) - _verify_archive(tmp_archive) - - TRANSFER_DIR.mkdir(parents=True, exist_ok=True) - dest = TRANSFER_DIR / archive_name - os.replace(tmp_archive, dest) - - finally: - tmp_archive.unlink(missing_ok=True) - shutil.rmtree(work_dir, ignore_errors=True) - - size = dest.stat().st_size - checksum = _sha256(dest) - logger.info(f"memory_transfer: exported {dest} ({size} bytes, sha256={checksum})") - return { - "filename": archive_name, - "size": size, - "checksum": checksum, - "record_count": record_count, - "components": components, - } - -def _verify_archive(path: Path, extract_to: Path | None = None) -> dict: - """Validate archive members and extract once for checksum verification.""" - if path.stat().st_size > _MAX_COMPRESSED_BYTES: - raise ValueError(f"Archive too large: {path.stat().st_size} bytes") - - seen_names: set[str] = set() - total_extracted = 0 - - with tarfile.open(path, "r:gz") as tar: - for member in tar.getmembers(): - name = member.name - if name not in _ALLOWLIST: - raise ValueError(f"Unexpected archive member: {name!r}") - if name in seen_names: - raise ValueError(f"Duplicate archive member: {name!r}") - seen_names.add(name) - if not member.isfile(): - raise ValueError(f"Non-regular member: {name!r}") - if ".." in Path(name).parts or Path(name).is_absolute(): - raise ValueError(f"Path traversal in member: {name!r}") - total_extracted += member.size - if total_extracted > _MAX_EXTRACTED_BYTES: - raise ValueError("Archive extracted size exceeds limit") - - manifest_file = tar.extractfile("manifest.json") - if manifest_file is None: - raise ValueError("manifest.json missing from archive") - manifest = _validate_manifest(json.loads(manifest_file.read())) - - if manifest["format_version"] != ARCHIVE_FORMAT_VERSION: - raise ValueError(f"Unsupported format_version: {manifest.get('format_version')}") - - components = manifest.get("components", []) - expected_members = {"manifest.json"} - expected_members.update(*(_COMPONENT_FILES[component] for component in components)) - if seen_names != expected_members: - raise ValueError("Archive members do not match manifest components") - - if extract_to is not None: - extract_to.mkdir(parents=True, exist_ok=True) - _safe_extract(tar, extract_to) - _validate_extracted_archive(extract_to, manifest) - else: - with tempfile.TemporaryDirectory() as tmp: - staging = Path(tmp) - _safe_extract(tar, staging) - _validate_extracted_archive(staging, manifest) - - return manifest - -def _safe_extract(tar: tarfile.TarFile, dest: Path) -> None: - """Extract only regular, allowlisted archive members without tarfile filters. - - Python 3.11 does not support TarFile.extractall(filter=...), so extraction - is performed explicitly after validating each member's fixed archive path. - """ - base = dest.resolve() - for member in tar.getmembers(): - name = member.name - if name not in _ALLOWLIST or not member.isfile(): - raise ValueError(f"Unsafe archive member: {name!r}") - target = (base / name).resolve() - if base not in target.parents: - raise ValueError(f"Path traversal in member: {name!r}") - target.parent.mkdir(parents=True, exist_ok=True) - source = tar.extractfile(member) - if source is None: - raise ValueError(f"Could not read archive member: {name!r}") - with source, target.open("wb") as output: - shutil.copyfileobj(source, output) - -def _parse_component_flags(args: argparse.Namespace) -> tuple[bool, bool]: - no_history = getattr(args, "no_history", False) - no_vector = getattr(args, "no_vector", False) or getattr(args, "only_history", False) - if no_history and no_vector: - raise ValueError("--no-history and --no-vector together import nothing. Aborting.") - return not no_history, not no_vector - -def _tx_marker(memory_base: Path) -> Path: - return memory_base / _TX_MARKER_NAME - -def _receipt_path(memory_base: Path, digest: str, mode: str, - include_history: bool, include_vectors: bool) -> Path: - components = "-".join(component for component, included in ( - ("history", include_history), ("ltm", include_vectors) - ) if included) - return memory_base / _RECEIPT_DIR_NAME / f"{digest}-{mode}-{components}.json" - -def _write_receipt(path: Path, digest: str, mode: str, - include_history: bool, include_vectors: bool) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - receipt = { - "archive_sha256": digest, - "mode": mode, - "include_history": include_history, - "include_vectors": include_vectors, - "imported_at": _utc_now(), - } - temporary = path.with_suffix(".tmp") - temporary.write_text(json.dumps(receipt, sort_keys=True), encoding="utf-8") - os.replace(temporary, path) - -def _marker_has_receipt(marker: Path, memory_base: Path) -> bool: - try: - receipt_name = json.loads(marker.read_text(encoding="utf-8")).get("receipt") - except (json.JSONDecodeError, OSError): - return False - return ( - isinstance(receipt_name, str) - and Path(receipt_name).name == receipt_name - and (memory_base / _RECEIPT_DIR_NAME / receipt_name).is_file() - ) - -def _append_state(include_history: bool, include_vectors: bool) -> dict: - history = history_path() - return { - "history": { - "existed": history.exists(), - "size": history.stat().st_size if history.exists() else 0, - } if include_history else None, - "import_id": uuid.uuid4().hex if include_vectors else None, - } - -def _rollback_append(state: dict) -> None: - history = state.get("history") - if isinstance(history, dict): - path = history_path() - if history.get("existed"): - if path.exists(): - with path.open("rb+") as output: - output.truncate(history["size"]) - else: - path.unlink(missing_ok=True) - - import_id = state.get("import_id") - if isinstance(import_id, str): - col = gateway._get_collection() - imported = col.get(where={"import_id": import_id}, include=[]) - if imported["ids"]: - col.delete(ids=imported["ids"]) - -def recover(memory_base: Path | None = None) -> None: - """Restore an interrupted import or fail when its rollback is unavailable.""" - base = memory_base or memory_dir_path() - marker = _tx_marker(base) - rollback = base / ".import_rollback" - - if not marker.exists(): - return - - if _marker_has_receipt(marker, base): - marker.unlink(missing_ok=True) - shutil.rmtree(rollback, ignore_errors=True) - logger.info("memory_transfer: completed import transaction recovered") - return - - try: - transaction = json.loads(marker.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - transaction = {} - append = transaction.get("append") if isinstance(transaction, dict) else None - if isinstance(append, dict): - logger.warning("memory_transfer: unfinished append import detected — removing partial import") - _rollback_append(append) - marker.unlink(missing_ok=True) - logger.info("memory_transfer: append transaction recovery complete") - return - - logger.warning("memory_transfer: unfinished import transaction detected — restoring rollback") - - if not rollback.exists(): - raise RuntimeError( - "Import transaction marker found but no rollback copy exists. " - "Cannot recover safely — operator intervention required." - ) - - _restore_rollback(rollback) - marker.unlink(missing_ok=True) - shutil.rmtree(rollback, ignore_errors=True) - logger.info("memory_transfer: transaction recovery complete") - -def _restore_rollback(rollback: Path) -> None: - hist_rb = rollback / "history.metta" - chroma_rb = rollback / "chroma_db" - state_path = rollback / _ROLLBACK_STATE_NAME - try: - state = json.loads(state_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - state = {} - - if state.get("history") is False: - history_path().unlink(missing_ok=True) - elif hist_rb.exists(): - history_path().parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(hist_rb, history_path()) - - live_chroma = chroma_db_path() - if state.get("vectors") is False: - shutil.rmtree(live_chroma, ignore_errors=True) - elif chroma_rb.exists(): - if live_chroma.exists(): - shutil.rmtree(live_chroma) - shutil.copytree(chroma_rb, live_chroma) - if "vectors" in state: - gateway._client = None - gateway._collection = None - -def import_archive(archive_path: Path, mode: str = "overwrite", - include_history: bool = True, - include_vectors: bool = True) -> None: - """Validate and restore a memory archive before the agent loop starts.""" - if mode not in ("overwrite", "append"): - raise ValueError(f"Invalid mode: {mode!r}. Use overwrite or append.") - if not archive_path.exists(): - raise FileNotFoundError(f"Archive not found: {archive_path}") - - logger.info(f"memory_transfer: importing {archive_path} (mode={mode})") - - base = memory_dir_path() - digest = _sha256(archive_path) - receipt = _receipt_path(base, digest, mode, include_history, include_vectors) - if receipt.exists(): - logger.info("memory_transfer: archive already imported; skipping") - return - staging = base / ".import_staging" - shutil.rmtree(staging, ignore_errors=True) - try: - manifest = _verify_archive(archive_path, staging) - import_history = ( - include_history - and "history" in manifest.get("components", []) - and (staging / "history" / "history.metta").is_file() - ) - import_vectors = ( - include_vectors - and "ltm" in manifest.get("components", []) - and (staging / "vector" / "records.jsonl").is_file() - ) - if mode == "overwrite": - _import_overwrite( - staging, manifest, import_history, import_vectors, receipt, digest - ) - else: - _import_append( - staging, manifest, import_history, import_vectors, receipt, digest - ) - finally: - shutil.rmtree(staging, ignore_errors=True) - - logger.info("memory_transfer: import complete") - -def _import_overwrite(staging: Path, manifest: dict, - include_history: bool, include_vectors: bool, - receipt: Path, digest: str) -> None: - """Overwrite live memory with rollback and crash-recovery protection.""" - base = memory_dir_path() - rollback = base / ".import_rollback" - marker = _tx_marker(base) - - shutil.rmtree(rollback, ignore_errors=True) - rollback.mkdir(parents=True) - state = {"history": history_path().exists() if include_history else None, - "vectors": chroma_db_path().exists() if include_vectors else None} - (rollback / _ROLLBACK_STATE_NAME).write_text( - json.dumps(state), encoding="utf-8" - ) - if include_history and state["history"]: - shutil.copy2(history_path(), rollback / "history.metta") - if include_vectors and state["vectors"]: - rb_chroma = rollback / "chroma_db" - shutil.copytree(chroma_db_path(), rb_chroma) - - marker.write_text(json.dumps({"receipt": receipt.name}), encoding="utf-8") - try: - if include_history: - src = staging / "history" / "history.metta" - history_path().parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, history_path()) - - if include_vectors: - _restore_vectors(staging, manifest) - - _smoke_test(include_history, include_vectors) - - except Exception: - logger.exception("memory_transfer: overwrite failed — restoring rollback") - try: - _restore_rollback(rollback) - except Exception: - logger.exception("memory_transfer: rollback itself failed — preserving marker and rollback for manual recovery") - raise - marker.unlink(missing_ok=True) - shutil.rmtree(rollback, ignore_errors=True) - raise - else: - _write_receipt(receipt, digest, "overwrite", include_history, include_vectors) - marker.unlink(missing_ok=True) - shutil.rmtree(rollback, ignore_errors=True) - -def _restore_vectors(staging: Path, manifest: dict) -> None: - """Restore user-memory vectors while preserving non-user records.""" - records_path = staging / "vector" / "records.jsonl" - if not records_path.exists(): - return - - col = gateway._get_collection() - target_dimension = _collection_dimension(col) - - existing = col.get(include=["metadatas"]) - user_ids = [ - eid for eid, meta in zip(existing["ids"], existing.get("metadatas") or []) - if _is_user_memory_record(meta) - ] - if user_ids: - col.delete(ids=user_ids) - - gateway._client = None - gateway._collection = None - col = gateway._get_collection() - - for records in _record_batches(records_path): - _reembed_records(records, manifest, target_dimension) - col.add( - ids =[record["id"] for record in records], - documents =[record["document"] for record in records], - embeddings =[record["embedding"] for record in records], - metadatas =[record["metadata"] for record in records], - ) - -def _import_append(staging: Path, manifest: dict, - include_history: bool, include_vectors: bool, - receipt: Path, digest: str) -> None: - """Append imported memory to existing live memory.""" - base = memory_dir_path() - marker = _tx_marker(base) - state = _append_state(include_history, include_vectors) - marker.write_text( - json.dumps({"receipt": receipt.name, "append": state}), encoding="utf-8" - ) - try: - if include_history: - src = staging / "history" / "history.metta" - gateway.append_history("\n" + src.read_text(encoding="utf-8")) - - if include_vectors: - records_path = staging / "vector" / "records.jsonl" - col = gateway._get_collection() - target_dimension = _collection_dimension(col) - import_id = state["import_id"] - for records in _record_batches(records_path): - _reembed_records(records, manifest, target_dimension) - col.upsert( - ids =[f"import-{import_id}-{record['id']}" for record in records], - documents =[record["document"] for record in records], - embeddings=[record["embedding"] for record in records], - metadatas=[{**record["metadata"], "import_id": import_id} - for record in records], - ) - _smoke_test(include_history, include_vectors) - except Exception: - logger.exception("memory_transfer: append failed — removing partial import") - try: - _rollback_append(state) - except Exception: - logger.exception("memory_transfer: append cleanup failed — preserving marker for recovery") - raise - marker.unlink(missing_ok=True) - raise - else: - _write_receipt(receipt, digest, "append", include_history, include_vectors) - marker.unlink(missing_ok=True) - -def _smoke_test(include_history: bool, include_vectors: bool) -> None: - if include_history: - history_path().read_text(encoding="utf-8")[:1] - if include_vectors: - gateway._get_collection().get(limit=1) - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="memory_transfer") - sub = parser.add_subparsers(dest="command", required=True) - - imp = sub.add_parser("import", help="Restore a memory archive (pre-start)") - imp.add_argument("archive", type=Path) - imp.add_argument("--mode", choices=["overwrite", "append"], default="overwrite") - imp.add_argument("--no-history", action="store_true") - imp.add_argument("--no-vector", action="store_true") - imp.add_argument("--only-history", action="store_true") - - sub.add_parser("recover", help="Recover from an interrupted import transaction") - return parser - -if __name__ == "__main__": - import sys - args = _build_parser().parse_args() - - if args.command == "recover": - recover() - sys.exit(0) - try: - inc_hist, inc_vec = _parse_component_flags(args) - import_archive(args.archive, mode=args.mode, - include_history=inc_hist, include_vectors=inc_vec) - except Exception as exc: - logger.error(f"memory_transfer import failed: {exc}") - sys.exit(1) diff --git a/tests/test_memory_gateway.py b/tests/test_memory_gateway.py index 3b601dcf..9cea3195 100644 --- a/tests/test_memory_gateway.py +++ b/tests/test_memory_gateway.py @@ -5,17 +5,13 @@ EMBEDDING = [0.1, 0.2, 0.3] - @pytest.fixture(autouse=True) def isolated_gateway(tmp_path, monkeypatch): monkeypatch.setenv("MEMORY_DIR", str(tmp_path)) - import src.memory_layout as layout - importlib.reload(layout) import src.memory_gateway as gateway importlib.reload(gateway) return gateway - def test_history_append_and_ltm_query(isolated_gateway, tmp_path): isolated_gateway.append_history("first") isolated_gateway.append_history("second") @@ -23,20 +19,15 @@ def test_history_append_and_ltm_query(isolated_gateway, tmp_path): assert (tmp_path / "history.metta").read_text() == "first\nsecond\n" assert isolated_gateway.query(EMBEDDING, 1) == [["2026-01-01", "portable fact"]] - metadata = isolated_gateway._get_collection().get(include=["metadatas"])["metadatas"] - assert metadata[0]["record_kind"] == "user_memory" - def test_export_lock_blocks_history_write(isolated_gateway, tmp_path): finished = threading.Event() thread = threading.Thread( target=lambda: (isolated_gateway.append_history("concurrent"), finished.set()) ) - with isolated_gateway._write_lock: thread.start() assert not finished.wait(0.05) thread.join(timeout=2) - assert finished.is_set() assert (tmp_path / "history.metta").read_text() == "concurrent\n" diff --git a/tests/test_memory_layout.py b/tests/test_memory_layout.py deleted file mode 100644 index fd64d676..00000000 --- a/tests/test_memory_layout.py +++ /dev/null @@ -1,19 +0,0 @@ -import importlib - -import src.memory_layout as memory_layout - - -def test_memory_dir_controls_both_persistent_paths(tmp_path, monkeypatch): - monkeypatch.setenv("MEMORY_DIR", str(tmp_path)) - layout = importlib.reload(memory_layout) - - assert layout.history_path() == tmp_path.resolve() / "history.metta" - assert layout.chroma_db_path() == tmp_path.resolve() / "chroma_db" - - -def test_default_paths_are_absolute(monkeypatch): - monkeypatch.delenv("MEMORY_DIR", raising=False) - layout = importlib.reload(memory_layout) - - assert layout.history_path().is_absolute() - assert layout.chroma_db_path().is_absolute() diff --git a/tests/test_memory_transfer.py b/tests/test_memory_transfer.py deleted file mode 100644 index ed82e2c6..00000000 --- a/tests/test_memory_transfer.py +++ /dev/null @@ -1,273 +0,0 @@ -import importlib -import io -import json -import tarfile -import threading -import types -from pathlib import Path - -import pytest -import yaml - -EMBEDDING = [0.1, 0.2, 0.3] -REPO_ROOT = Path(__file__).resolve().parents[1] - - -@pytest.fixture(autouse=True) -def isolated_memory(tmp_path, monkeypatch): - memory_dir = tmp_path / "memory" - memory_dir.mkdir() - monkeypatch.setenv("MEMORY_DIR", str(memory_dir)) - - import src.memory_layout as layout - importlib.reload(layout) - import src.memory_gateway as gateway - importlib.reload(gateway) - import src.memory_transfer as transfer - importlib.reload(transfer) - transfer.TRANSFER_DIR = tmp_path / "transfer" - transfer.TRANSFER_DIR.mkdir() - return transfer, gateway, memory_dir - - -def test_launcher_and_policy_wiring(): - launcher = (REPO_ROOT / "scripts" / "omegaclaw").read_text() - entrypoint = (REPO_ROOT / "entrypoint.sh").read_text() - config = (REPO_ROOT / "config" / "config.yaml").read_text() - dockerfile = (REPO_ROOT / "Dockerfile").read_text() - policy = yaml.safe_load((REPO_ROOT / "profile" / "policy.yaml").read_text()) - - assert "memoryExportEnabled: false" in config - assert "--enable-memory-export" in launcher - assert "MEMORY_IMPORT_ONLY_HISTORY" in launcher - assert ' -- sh "$OMEGACLAW_DIR" "${import_args[@]}"' in entrypoint - assert 'echo "memory_transfer: import complete"' in entrypoint - assert "mkdir -p /memory-transfer" in dockerfile - assert "/memory-transfer" in policy["filesystem_policy"]["read_write"] - - -def test_export_both_contains_only_portable_user_memory(isolated_memory): - transfer, gateway, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("portable history\n") - gateway.remember("portable fact", EMBEDDING, "2026-01-01") - gateway._get_collection().add( - ids=["knowledge"], documents=["non-user data"], embeddings=[EMBEDDING], - metadatas=[{"type": "chunk"}], - ) - - result = transfer.export("both") - with tarfile.open(transfer.TRANSFER_DIR / result["filename"], "r:gz") as archive: - assert set(archive.getnames()) == { - "manifest.json", "history/history.metta", "vector/collections.json", "vector/records.jsonl" - } - records = [json.loads(line) for line in archive.extractfile("vector/records.jsonl")] - - assert result["record_count"] == 1 - assert [record["document"] for record in records] == ["portable fact"] - - -def test_concurrent_exports_publish_distinct_archives(isolated_memory): - transfer, _, _ = isolated_memory - results = [] - threads = [threading.Thread(target=lambda: results.append(transfer.export("history"))) for _ in range(2)] - for thread in threads: - thread.start() - for thread in threads: - thread.join(timeout=5) - - assert all(not thread.is_alive() for thread in threads) - assert len({result["filename"] for result in results}) == 2 - - -def test_malformed_archive_is_rejected_before_mutating_memory(isolated_memory, tmp_path): - transfer, _, memory_dir = isolated_memory - history = memory_dir / "history.metta" - history.write_text("original\n") - valid = transfer.TRANSFER_DIR / transfer.export("history")["filename"] - invalid = tmp_path / "invalid.tar.gz" - with tarfile.open(valid, "r:gz") as source, tarfile.open(invalid, "w:gz") as output: - for member in source.getmembers(): - output.addfile(member, source.extractfile(member)) - extra = tarfile.TarInfo("unexpected.txt") - extra.size = 1 - output.addfile(extra, io.BytesIO(b"x")) - - history.write_text("live\n") - with pytest.raises(ValueError, match="Unexpected archive member"): - transfer.import_archive(invalid) - assert history.read_text() == "live\n" - - -def test_invalid_record_count_is_rejected_before_mutating_vectors(isolated_memory, tmp_path): - transfer, gateway, _ = isolated_memory - gateway.remember("source fact", EMBEDDING, "2026-01-01") - valid = transfer.TRANSFER_DIR / transfer.export("ltm")["filename"] - invalid = tmp_path / "invalid-count.tar.gz" - with tarfile.open(valid, "r:gz") as source, tarfile.open(invalid, "w:gz") as output: - for member in source.getmembers(): - if member.name == "manifest.json": - manifest = json.loads(source.extractfile(member).read()) - manifest["record_count"] = 2 - data = json.dumps(manifest).encode() - member.size = len(data) - output.addfile(member, io.BytesIO(data)) - else: - output.addfile(member, source.extractfile(member)) - - with pytest.raises(ValueError, match="record count"): - transfer.import_archive(invalid, include_history=False, include_vectors=True) - assert gateway._get_collection().count() == 1 - - -def test_invalid_manifest_schema_is_rejected_before_mutating_memory(isolated_memory, tmp_path): - transfer, _, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("archive history\n") - valid = transfer.TRANSFER_DIR / transfer.export("history")["filename"] - invalid = tmp_path / "invalid-manifest.tar.gz" - with tarfile.open(valid, "r:gz") as source, tarfile.open(invalid, "w:gz") as output: - for member in source.getmembers(): - if member.name == "manifest.json": - manifest = json.loads(source.extractfile(member).read()) - manifest["created_at"] = 1 - data = json.dumps(manifest).encode() - member.size = len(data) - output.addfile(member, io.BytesIO(data)) - else: - output.addfile(member, source.extractfile(member)) - - (memory_dir / "history.metta").write_text("live history\n") - with pytest.raises(ValueError, match="created_at"): - transfer.import_archive(invalid) - assert (memory_dir / "history.metta").read_text() == "live history\n" - - -def test_import_extracts_archive_once(isolated_memory, monkeypatch): - transfer, _, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("archive history\n") - archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] - extractions = [] - original_extract = transfer._safe_extract - - def track_extract(tar, dest): - extractions.append(dest) - original_extract(tar, dest) - - monkeypatch.setattr(transfer, "_safe_extract", track_extract) - transfer.import_archive(archive) - - assert len(extractions) == 1 - - -def test_overwrite_round_trip_restores_history_and_ltm(isolated_memory): - transfer, gateway, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("archived history\n") - gateway.remember("archived fact", EMBEDDING, "2026-01-01") - collection = gateway._get_collection() - collection.add( - ids=["knowledge"], documents=["retain this"], embeddings=[EMBEDDING], metadatas=[{"type": "chunk"}] - ) - archive = transfer.TRANSFER_DIR / transfer.export("both")["filename"] - - (memory_dir / "history.metta").write_text("live history\n") - collection.delete(ids=[record_id for record_id in collection.get(include=[])["ids"] if record_id != "knowledge"]) - transfer.import_archive(archive, mode="overwrite") - - documents = gateway._get_collection().get(include=["documents"])["documents"] - assert (memory_dir / "history.metta").read_text() == "archived history\n" - assert set(documents) == {"archived fact", "retain this"} - - -def test_history_only_import_does_not_open_chromadb(isolated_memory, monkeypatch): - transfer, gateway, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("archived history\n") - archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] - monkeypatch.setattr(gateway, "_get_collection", lambda: pytest.fail("opened ChromaDB")) - - transfer.import_archive(archive, include_history=True, include_vectors=False) - - assert (memory_dir / "history.metta").read_text() == "archived history\n" - - -def test_reembedding_runs_for_changed_embedding_profile(isolated_memory, monkeypatch): - transfer, gateway, _ = isolated_memory - gateway.remember("portable fact", EMBEDDING, "2026-01-01") - archive = transfer.TRANSFER_DIR / transfer.export("ltm")["filename"] - monkeypatch.setattr(transfer, "_embedding_profile", lambda: {"provider": "Local", "model": "new"}) - rag = types.ModuleType("src.rag") - rag.local_embed_batch = lambda documents: [[0.4, 0.5, 0.6] for _ in documents] - rag.openai_embed_batch = rag.local_embed_batch - monkeypatch.setitem(__import__("sys").modules, "src.rag", rag) - - transfer.import_archive(archive, include_history=False, include_vectors=True) - - embedding = gateway._get_collection().get(include=["embeddings"])["embeddings"][0] - assert list(embedding) == pytest.approx([0.4, 0.5, 0.6]) - - -def test_overwrite_failure_restores_absent_history_state(isolated_memory, monkeypatch): - transfer, _, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("archive history\n") - archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] - (memory_dir / "history.metta").unlink() - monkeypatch.setattr(transfer, "_smoke_test", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) - - with pytest.raises(RuntimeError, match="boom"): - transfer.import_archive(archive) - assert not (memory_dir / "history.metta").exists() - - -def test_receipt_prevents_repeat_overwrite(isolated_memory): - transfer, _, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("archive history\n") - archive = transfer.TRANSFER_DIR / transfer.export("history")["filename"] - (memory_dir / "history.metta").write_text("before import\n") - transfer.import_archive(archive) - (memory_dir / "history.metta").write_text("new memory\n") - - transfer.import_archive(archive) - - assert (memory_dir / "history.metta").read_text() == "new memory\n" - - -def test_append_failure_removes_partial_history_and_vectors(isolated_memory, monkeypatch): - transfer, gateway, memory_dir = isolated_memory - (memory_dir / "history.metta").write_text("archive history\n") - gateway.remember("archive fact", EMBEDDING, "2026-01-01") - archive = transfer.TRANSFER_DIR / transfer.export("both")["filename"] - (memory_dir / "history.metta").write_text("live history\n") - gateway.remember("live fact", EMBEDDING, "2026-01-02") - original_count = gateway._get_collection().count() - monkeypatch.setattr(transfer, "_smoke_test", lambda *_: (_ for _ in ()).throw(RuntimeError("boom"))) - - with pytest.raises(RuntimeError, match="boom"): - transfer.import_archive(archive, mode="append") - - assert (memory_dir / "history.metta").read_text() == "live history\n" - assert gateway._get_collection().count() == original_count - - -def test_recovery_removes_interrupted_append(isolated_memory): - transfer, gateway, memory_dir = isolated_memory - history = memory_dir / "history.metta" - history.write_text("original\npartial\n") - import_id = "interrupted" - gateway._get_collection().add( - ids=["partial"], documents=["partial fact"], embeddings=[EMBEDDING], - metadatas=[{"import_id": import_id}], - ) - (memory_dir / transfer._TX_MARKER_NAME).write_text(json.dumps({"append": { - "history": {"existed": True, "size": len("original\n")}, "import_id": import_id, - }})) - - transfer.recover(memory_dir) - - assert history.read_text() == "original\n" - assert gateway._get_collection().get(where={"import_id": import_id}, include=[])["ids"] == [] - - -def test_export_is_disabled_until_explicitly_enabled(isolated_memory, monkeypatch): - transfer, _, _ = isolated_memory - monkeypatch.delenv("OMEGACLAW_memoryExportEnabled", raising=False) - assert transfer.is_export_enabled() is False - monkeypatch.setenv("OMEGACLAW_memoryExportEnabled", "true") - assert transfer.is_export_enabled() is True From 333379e0486c63006c2269c37253934094a3c595 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Wed, 19 Aug 2026 22:10:49 +0300 Subject: [PATCH 08/34] refactor(channels): centralize memory export dispatch --- channels/irc.py | 14 ++---- channels/mattermost.py | 14 ++---- ...ory_export_handler.py => memory_export.py} | 47 ++++++++++++------- channels/slack.py | 14 ++---- channels/telegram.py | 15 ++---- channels/wschat.py | 3 +- src/channels.py | 14 ++++++ ...xport_handler.py => test_memory_export.py} | 41 ++++++++++------ 8 files changed, 85 insertions(+), 77 deletions(-) rename channels/{memory_export_handler.py => memory_export.py} (80%) rename tests/{test_memory_export_handler.py => test_memory_export.py} (68%) diff --git a/channels/irc.py b/channels/irc.py index b3512304..f924bab5 100644 --- a/channels/irc.py +++ b/channels/irc.py @@ -9,7 +9,6 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key -from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -170,16 +169,9 @@ def _irc_session(channel, server, port, nick): msg = trailing.split(" :", 1)[1] state = _is_allowed_message(sender_nick, msg) if state == "allow": - if is_export_command(msg): - owner_key = f"irc:{_normalize_nick(sender_nick)}" - reply = handle_export_command( - msg, - owner_key, - lambda message, target=sender_nick: _send_export_reply(target, message), - ) - if reply is not None: - _send_export_reply(sender_nick, reply) - else: + owner_key = f"irc:{_normalize_nick(sender_nick)}" + deliver_reply = lambda message, target=sender_nick: _send_export_reply(target, message) + if not channels.handle_control_message(msg, owner_key, deliver_reply): _set_last(f"{sender_nick}: {msg}") elif state == "auth_bound": send_message(f"Authentication successful for {sender_nick}.") diff --git a/channels/mattermost.py b/channels/mattermost.py index 6aaba97a..8e22144c 100644 --- a/channels/mattermost.py +++ b/channels/mattermost.py @@ -10,7 +10,6 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key -from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -182,16 +181,9 @@ def _ws_session(): state = _is_allowed_message(user_id, message) if state == "allow": name = _get_display_name(user_id) - if is_export_command(message): - owner_key = f"mattermost:{CHANNEL_ID}:{user_id}" - reply = handle_export_command( - message, - owner_key, - lambda text, target=user_id: _send_export_reply(target, text), - ) - if reply is not None: - _send_export_reply(user_id, reply) - else: + owner_key = f"mattermost:{CHANNEL_ID}:{user_id}" + deliver_reply = lambda text, target=user_id: _send_export_reply(target, text) + if not channels.handle_control_message(message, owner_key, deliver_reply): _set_last(f"{name}: {message}") elif state == "auth_bound": name = _get_display_name(user_id) diff --git a/channels/memory_export_handler.py b/channels/memory_export.py similarity index 80% rename from channels/memory_export_handler.py rename to channels/memory_export.py index d3492a29..cc235344 100644 --- a/channels/memory_export_handler.py +++ b/channels/memory_export.py @@ -1,15 +1,34 @@ """Shared authenticated /memory-export command handling.""" +import os import secrets import threading import time +from pathlib import Path import auth +from config import config_get_by_key from src.logger import get_logger -from src.memory_transfer import get_export_status, is_export_enabled, start_export_job +from memory_portability import MemoryTransfer logger = get_logger(__name__) +_TRANSFER_DIR = Path(os.environ.get("MEMORY_TRANSFER_DIR", "/memory-transfer")) + +_transfer = None + +def _get_transfer() -> MemoryTransfer: + global _transfer + if _transfer is None: + _transfer = MemoryTransfer(_TRANSFER_DIR) + return _transfer + +def start_export_job(component, on_complete=None): + return _get_transfer().start_export_job(component, on_complete=on_complete) + +def get_export_status(job_id): + return _get_transfer().get_export_status(job_id) + _TOKEN_TTL_SECONDS = 60 _token_lock = threading.Lock() @@ -18,8 +37,15 @@ _VALID_COMPONENTS = ("history", "ltm", "both") +def is_export_enabled() -> bool: + value = os.environ.get("OMEGACLAW_memoryExportEnabled") + if value is None: + value = config_get_by_key("memoryExportEnabled", False) + return value is True or ( + isinstance(value, str) and value.strip().lower() == "true" + ) + def is_export_command(text: str) -> bool: - """Return whether text is reserved for the memory-export control plane.""" command = text.strip().split(None, 1) if not command: return False @@ -27,12 +53,10 @@ def is_export_command(text: str) -> bool: return name == "/memory-export" or name.startswith("/memory-export@") def _command_arguments(text: str) -> str: - """Return arguments after the plain or Telegram-qualified command name.""" parts = text.strip().split(None, 1) return parts[1].strip() if len(parts) == 2 else "" def _issue_token(owner_key: str, component: str) -> str: - """Generate and store an owner-scoped confirmation token.""" token = secrets.token_hex(8) _pending_requests[owner_key] = ( token, @@ -46,15 +70,6 @@ def handle_export_command( owner_key: str = "default-owner", deliver_completion=lambda _message: None, ) -> str | None: - """ - Parse and handle a /memory-export command from the authenticated owner. - - Returns a reply string to send back to the owner, or None when policy - disables export. Callers must use is_export_command() to consume all - reserved control commands before they can reach the LLM. - - text: the raw message text, e.g. "/memory-export history" - """ stripped = text.strip() if not is_export_command(stripped): @@ -89,7 +104,7 @@ def handle_export_command( def _handle_request(owner_key: str, component: str) -> str: with _token_lock: token = _issue_token(owner_key, component) - logger.info(f"memory_export_handler: issued confirmation token for component={component}") + logger.info(f"memory_export: issued confirmation token for component={component}") return ( f"Export requested for: {component}\n" f"Confirm within {_TOKEN_TTL_SECONDS}s:\n" @@ -120,10 +135,10 @@ def _handle_confirm(owner_key: str, token: str, deliver_completion) -> str: ), ) except Exception as exc: - logger.exception(f"memory_export_handler: failed to start export job: {exc}") + logger.exception(f"memory_export: failed to start export job: {exc}") return f"Export failed to start: {exc}" - logger.info(f"memory_export_handler: export job {job_id} started (component={component})") + logger.info(f"memory_export: export job {job_id} started (component={component})") with _token_lock: _job_owners[job_id] = owner_key return ( diff --git a/channels/slack.py b/channels/slack.py index 950eccad..ad0a87a4 100644 --- a/channels/slack.py +++ b/channels/slack.py @@ -11,7 +11,6 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key -from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -362,16 +361,9 @@ def _poll_channel(channel_id): state = _is_allowed_message(channel_id, user_id, text) display_name = _get_display_name(user_id) if state == "allow": - if is_export_command(text): - owner_key = f"slack:{channel_id}:{user_id}" - reply = handle_export_command( - text, - owner_key, - lambda message, target=user_id: _send_export_reply(target, message), - ) - if reply is not None: - _send_export_reply(user_id, reply) - else: + owner_key = f"slack:{channel_id}:{user_id}" + deliver_reply = lambda message, target=user_id: _send_export_reply(target, message) + if not channels.handle_control_message(text, owner_key, deliver_reply): _set_last(f"{display_name}: {text}") elif state == "auth_bound": send_message(f"Authentication successful for {display_name}.") diff --git a/channels/telegram.py b/channels/telegram.py index 129a553e..c8cbbfd9 100644 --- a/channels/telegram.py +++ b/channels/telegram.py @@ -9,7 +9,6 @@ from delivery_queue import PendingMessages import channels from config import config_get_by_key -from memory_export_handler import handle_export_command, is_export_command logger = get_logger(__name__) @@ -221,18 +220,10 @@ def _poll_loop(): state = _is_allowed_message(chat_id, user_id, text) display_name = _display_name(user, chat) - export_command = is_export_command(text) if state == "allow": - if export_command: - owner_key = f"telegram:{chat_id}:{user_id}" - reply = handle_export_command( - text, - owner_key, - lambda message, target=user_id: _send_export_reply(target, message), - ) - if reply is not None: - _send_export_reply(user_id, reply) - else: + owner_key = f"telegram:{chat_id}:{user_id}" + deliver_reply = lambda message, target=user_id: _send_export_reply(target, message) + if not channels.handle_control_message(text, owner_key, deliver_reply): _set_last(f"{display_name}: {text}") elif state == "auth_bound": send_message(f"Authentication successful for {display_name}.") diff --git a/channels/wschat.py b/channels/wschat.py index 968b237d..77d9fff1 100644 --- a/channels/wschat.py +++ b/channels/wschat.py @@ -64,7 +64,6 @@ from pathlib import Path import sys from config import config_get_by_key -from memory_export_handler import is_export_command _REPO_ROOT = Path(__file__).resolve().parents[1] if str(_REPO_ROOT) not in sys.path: @@ -195,7 +194,7 @@ def _handle_frame(raw_message): if not isinstance(seq, int) or not isinstance(text, str): logger.warning(f"Ignoring malformed user_message frame: {frame!r}") return - if is_export_command(text): + if channels.is_control_command(text): logger.info("Ignoring unavailable memory-export command on WebSocket channel") return _enqueue_user_message(seq, text) diff --git a/src/channels.py b/src/channels.py index ff3685ed..c75f7f48 100644 --- a/src/channels.py +++ b/src/channels.py @@ -4,6 +4,20 @@ _commChannelRegistry = {} +def is_control_command(text: str) -> bool: + from memory_export import is_export_command + return is_export_command(text) + +def handle_control_message(text: str, owner_key: str, deliver_reply) -> bool: + from memory_export import handle_export_command + + if not is_control_command(text): + return False + reply = handle_export_command(text, owner_key, deliver_reply) + if reply is not None: + deliver_reply(reply) + return True + class CommChannel: """Communication channel implementation""" diff --git a/tests/test_memory_export_handler.py b/tests/test_memory_export.py similarity index 68% rename from tests/test_memory_export_handler.py rename to tests/test_memory_export.py index a3273cc1..a522d503 100644 --- a/tests/test_memory_export_handler.py +++ b/tests/test_memory_export.py @@ -8,28 +8,29 @@ REPO_ROOT = Path(__file__).resolve().parents[1] - @pytest.fixture def handler(monkeypatch): auth = types.ModuleType("auth") auth.is_auth_enabled = lambda: True monkeypatch.setitem(sys.modules, "auth", auth) - logger = types.ModuleType("src.logger") - logger.get_logger = lambda name: __import__("logging").getLogger(name) - monkeypatch.setitem(sys.modules, "src.logger", logger) - transfer = types.ModuleType("src.memory_transfer") - transfer.is_export_enabled = lambda: True - transfer.start_export_job = lambda component, on_complete=None: "job-1" - transfer.get_export_status = lambda job_id: {"status": "unknown"} - monkeypatch.setitem(sys.modules, "src.memory_transfer", transfer) + + logger_mod = types.ModuleType("src.logger") + logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) + monkeypatch.setitem(sys.modules, "src.logger", logger_mod) + + mp_mod = types.ModuleType("memory_portability") + mp_mod.MemoryTransfer = object + monkeypatch.setitem(sys.modules, "memory_portability", mp_mod) + spec = importlib.util.spec_from_file_location( - "memory_export_handler_under_test", REPO_ROOT / "channels" / "memory_export_handler.py" + "memory_export_under_test", + REPO_ROOT / "channels" / "memory_export.py", ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + module.is_export_enabled = lambda: True return module - def test_export_command_requires_auth_and_policy(handler): handler.auth.is_auth_enabled = lambda: False assert handler.handle_export_command("/memory-export both") is None @@ -37,7 +38,6 @@ def test_export_command_requires_auth_and_policy(handler): handler.is_export_enabled = lambda: False assert handler.handle_export_command("/memory-export both") is None - def test_confirmation_starts_only_the_requested_export(handler): started = [] handler.start_export_job = lambda component, on_complete: started.append(component) or "job-1" @@ -47,7 +47,6 @@ def test_confirmation_starts_only_the_requested_export(handler): assert "job-1" in handler.handle_export_command(f"/memory-export confirm {token}") assert started == ["ltm"] - def test_expired_and_other_owner_tokens_cannot_start_export(handler): token = handler.handle_export_command("/memory-export history", "owner-a").split()[-1] assert "No pending export" in handler.handle_export_command( @@ -59,7 +58,6 @@ def test_expired_and_other_owner_tokens_cannot_start_export(handler): f"/memory-export confirm {token}", "owner-a" ).lower() - def test_completion_and_status_are_limited_to_requesting_owner(handler): delivered = [] @@ -73,3 +71,18 @@ def start_job(component, callback): assert "memory.tar.gz" in delivered[0] assert "unknown job ID" in handler.handle_export_command("/memory-export status job-1", "owner-b") + +def test_shared_dispatcher_consumes_control_commands(monkeypatch): + control = types.ModuleType("memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda text, owner, deliver: "Export requested" + monkeypatch.setitem(sys.modules, "memory_export", control) + + from src import channels + + replies = [] + assert channels.handle_control_message( + "/memory-export both", "telegram:chat:user", replies.append + ) + assert replies == ["Export requested"] + assert not channels.handle_control_message("hello", "telegram:chat:user", replies.append) From 10c7ea12ab15a0d8246ff7ca0a0315bf1a938c86 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 11:08:29 +0300 Subject: [PATCH 09/34] build(memory): use unified portability package Install import-kb v0.2.1 in the image and invoke its memory portability module during startup. This replaces the separate memory-portability package reference. --- Dockerfile | 9 +++++---- entrypoint.sh | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95de4603..b6736f73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -58,6 +58,7 @@ RUN python3 -m pip install --no-cache-dir --break-system-packages \ --extra-index-url https://pypi.org/simple/ \ torch==2.12.1 \ && python3 -m pip install --no-cache-dir --break-system-packages -r /tmp/requirements.txt + # Pre-download the sentence-transformers model so runtime does not need network access. RUN mkdir -p "${HF_HOME}" "${SENTENCE_TRANSFORMERS_HOME}" \ && python3 - <&2; exit 1; } @@ -69,7 +69,7 @@ if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then [[ "${MEMORY_IMPORT_NO_VECTOR:-0}" == "1" ]] && import_args+=(--no-vector) [[ "${MEMORY_IMPORT_ONLY_HISTORY:-0}" == "1" ]] && import_args+=(--no-vector) echo "memory_portability: importing ${MEMORY_IMPORT_FILE}" - su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m memory_portability import "$@"' \ + su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m import_knowledge.memory_portability import "$@"' \ -- sh "$OMEGACLAW_DIR" "${import_args[@]}" \ || { echo "Memory import failed. Aborting startup." >&2; exit 1; } echo "memory_portability: import complete" From 71ed9e429837b584da411fe14aeeaa5acb5748b1 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 11:08:51 +0300 Subject: [PATCH 10/34] refactor(memory-export): process exports in the main loop Queue confirmed export requests in channel handlers and process them before the next agent prompt. This keeps commands outside the LLM while making archive creation synchronous with normal memory writes. --- channels/memory_export.py | 95 ++++++++-------------------- docs/reference-memory-portability.md | 6 +- pyproject.toml | 1 - src/channels.py | 4 ++ src/loop.metta | 1 + tests/test_memory_export.py | 58 ++++++++++------- 6 files changed, 70 insertions(+), 95 deletions(-) diff --git a/channels/memory_export.py b/channels/memory_export.py index cc235344..b72755aa 100644 --- a/channels/memory_export.py +++ b/channels/memory_export.py @@ -1,5 +1,6 @@ """Shared authenticated /memory-export command handling.""" +from collections import deque import os import secrets import threading @@ -9,7 +10,7 @@ import auth from config import config_get_by_key from src.logger import get_logger -from memory_portability import MemoryTransfer +from import_knowledge.memory_portability import MemoryTransfer logger = get_logger(__name__) @@ -23,17 +24,11 @@ def _get_transfer() -> MemoryTransfer: _transfer = MemoryTransfer(_TRANSFER_DIR) return _transfer -def start_export_job(component, on_complete=None): - return _get_transfer().start_export_job(component, on_complete=on_complete) - -def get_export_status(job_id): - return _get_transfer().get_export_status(job_id) - _TOKEN_TTL_SECONDS = 60 -_token_lock = threading.Lock() +_request_lock = threading.Lock() _pending_requests: dict[str, tuple[str, str, float]] = {} -_job_owners: dict[str, str] = {} +_export_requests = deque() _VALID_COMPONENTS = ("history", "ltm", "both") @@ -91,18 +86,14 @@ def handle_export_command( if sub == "confirm": return _handle_confirm(owner_key, arg, deliver_completion) - if sub == "status": - return _handle_status(owner_key, arg) - return ( "Unknown /memory-export command. " "Use: /memory-export history|ltm|both or " - "/memory-export confirm or " - "/memory-export status " + "/memory-export confirm " ) def _handle_request(owner_key: str, component: str) -> str: - with _token_lock: + with _request_lock: token = _issue_token(owner_key, component) logger.info(f"memory_export: issued confirmation token for component={component}") return ( @@ -115,7 +106,7 @@ def _handle_confirm(owner_key: str, token: str, deliver_completion) -> str: if not token: return "Usage: /memory-export confirm " - with _token_lock: + with _request_lock: pending = _pending_requests.get(owner_key) if pending is None: return "No pending export request. Start with /memory-export history|ltm|both" @@ -127,60 +118,28 @@ def _handle_confirm(owner_key: str, token: str, deliver_completion) -> str: return "Invalid token." del _pending_requests[owner_key] + with _request_lock: + _export_requests.append((component, deliver_completion)) + return "Export queued. It will run in the next agent iteration." + +def process_pending_export() -> None: + with _request_lock: + if not _export_requests: + return + component, deliver_completion = _export_requests.popleft() try: - job_id = start_export_job( - component, - lambda completed_job_id, status: deliver_completion( - _format_completion(completed_job_id, status) - ), - ) + result = _get_transfer().export(component) + reply = _format_export(result) except Exception as exc: - logger.exception(f"memory_export: failed to start export job: {exc}") - return f"Export failed to start: {exc}" + logger.exception(f"memory_export: export failed: {exc}") + reply = f"Memory export failed: {exc}" + deliver_completion(reply) - logger.info(f"memory_export: export job {job_id} started (component={component})") - with _token_lock: - _job_owners[job_id] = owner_key +def _format_export(result: dict) -> str: return ( - f"Export started. Job ID: {job_id}\n" - f"Check progress: /memory-export status {job_id}" + "Memory export complete\n" + f"File: {result.get('filename')}\n" + f"Size: {result.get('size')} bytes\n" + f"SHA-256: {result.get('checksum')}\n" + f"Records: {result.get('record_count')}" ) - -def _handle_status(owner_key: str, job_id: str) -> str: - if not job_id: - return "Usage: /memory-export status " - - with _token_lock: - if _job_owners.get(job_id) != owner_key: - return f"Export {job_id}: unknown job ID" - - status = get_export_status(job_id) - state = status.get("status", "unknown") - - if state == "running": - return f"Export {job_id}: running…" - - if state == "done": - return ( - f"Export {job_id}: done\n" - f"File: {status.get('filename')}\n" - f"Size: {status.get('size')} bytes\n" - f"SHA-256: {status.get('checksum')}\n" - f"Records: {status.get('record_count')}" - ) - - if state == "failed": - return f"Export {job_id}: failed — {status.get('error')}" - - return f"Export {job_id}: unknown job ID" - -def _format_completion(job_id: str, status: dict) -> str: - if status.get("status") == "done": - return ( - f"Export {job_id}: done\n" - f"File: {status.get('filename')}\n" - f"Size: {status.get('size')} bytes\n" - f"SHA-256: {status.get('checksum')}\n" - f"Records: {status.get('record_count')}" - ) - return f"Export {job_id}: failed — {status.get('error')}" diff --git a/docs/reference-memory-portability.md b/docs/reference-memory-portability.md index c46d39a8..0fb025fb 100644 --- a/docs/reference-memory-portability.md +++ b/docs/reference-memory-portability.md @@ -29,11 +29,11 @@ the returned short-lived token: /memory-export ltm /memory-export both /memory-export confirm -/memory-export status ``` -The export runs in the background. Completion is delivered only to the owner -who started it and includes the filename, record count, size, and SHA-256. +The confirmed export runs in the next agent iteration. Completion is delivered +only to the owner who started it and includes the filename, record count, size, +and SHA-256. Archives contain selected persistent user memory only: diff --git a/pyproject.toml b/pyproject.toml index 2cfa940f..748eaf71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,4 @@ [tool.pytest.ini_options] pythonpath = [ - ".", "src" ] diff --git a/src/channels.py b/src/channels.py index c75f7f48..e0becd0a 100644 --- a/src/channels.py +++ b/src/channels.py @@ -18,6 +18,10 @@ def handle_control_message(text: str, owner_key: str, deliver_reply) -> bool: deliver_reply(reply) return True +def process_control_messages() -> None: + from memory_export import process_pending_export + process_pending_export() + class CommChannel: """Communication channel implementation""" diff --git a/src/loop.metta b/src/loop.metta index eced53cd..3db13bd8 100644 --- a/src/loop.metta +++ b/src/loop.metta @@ -68,6 +68,7 @@ (commChannelSend (version)) (llmProviderStart (provider))) (change-state! &loops (- (get-state &loops) 1))) + (py-call (channels.process_control_messages)) (let $prompt (getContext) (progn (log INFO "loop" (---------iteration $k)) (heartbeat $k) diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index a522d503..0d0be87c 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -1,3 +1,4 @@ +import importlib import importlib.util import sys import time @@ -18,9 +19,12 @@ def handler(monkeypatch): logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) monkeypatch.setitem(sys.modules, "src.logger", logger_mod) - mp_mod = types.ModuleType("memory_portability") + import_knowledge = types.ModuleType("import_knowledge") + import_knowledge.__path__ = [] + monkeypatch.setitem(sys.modules, "import_knowledge", import_knowledge) + mp_mod = types.ModuleType("import_knowledge.memory_portability") mp_mod.MemoryTransfer = object - monkeypatch.setitem(sys.modules, "memory_portability", mp_mod) + monkeypatch.setitem(sys.modules, "import_knowledge.memory_portability", mp_mod) spec = importlib.util.spec_from_file_location( "memory_export_under_test", @@ -38,17 +42,11 @@ def test_export_command_requires_auth_and_policy(handler): handler.is_export_enabled = lambda: False assert handler.handle_export_command("/memory-export both") is None -def test_confirmation_starts_only_the_requested_export(handler): - started = [] - handler.start_export_job = lambda component, on_complete: started.append(component) or "job-1" - token = handler.handle_export_command("/memory-export ltm").split()[-1] - - assert "Invalid token" in handler.handle_export_command("/memory-export confirm wrong") - assert "job-1" in handler.handle_export_command(f"/memory-export confirm {token}") - assert started == ["ltm"] - def test_expired_and_other_owner_tokens_cannot_start_export(handler): token = handler.handle_export_command("/memory-export history", "owner-a").split()[-1] + assert "Invalid token" in handler.handle_export_command( + "/memory-export confirm wrong", "owner-a" + ) assert "No pending export" in handler.handle_export_command( f"/memory-export confirm {token}", "owner-b" ) @@ -58,27 +56,39 @@ def test_expired_and_other_owner_tokens_cannot_start_export(handler): f"/memory-export confirm {token}", "owner-a" ).lower() -def test_completion_and_status_are_limited_to_requesting_owner(handler): +def test_export_completion_is_delivered_after_loop_processing(handler): delivered = [] - - def start_job(component, callback): - callback("job-1", {"status": "done", "filename": "memory.tar.gz"}) - return "job-1" - - handler.start_export_job = start_job + exported = [] + handler._get_transfer = lambda: types.SimpleNamespace( + export=lambda component: exported.append(component) or { + "filename": "memory.tar.gz", + "size": 1, + "checksum": "abc", + "record_count": 1, + } + ) token = handler.handle_export_command("/memory-export both", "owner-a", delivered.append).split()[-1] - handler.handle_export_command(f"/memory-export confirm {token}", "owner-a", delivered.append) - - assert "memory.tar.gz" in delivered[0] - assert "unknown job ID" in handler.handle_export_command("/memory-export status job-1", "owner-b") + delivered.append( + handler.handle_export_command( + f"/memory-export confirm {token}", "owner-a", delivered.append + ) + ) + assert delivered == ["Export queued. It will run in the next agent iteration."] + assert exported == [] + handler.process_pending_export() + assert exported == ["both"] + assert "memory.tar.gz" in delivered[-1] def test_shared_dispatcher_consumes_control_commands(monkeypatch): control = types.ModuleType("memory_export") control.is_export_command = lambda text: text == "/memory-export both" control.handle_export_command = lambda text, owner, deliver: "Export requested" + processed = [] + control.process_pending_export = lambda: processed.append(True) monkeypatch.setitem(sys.modules, "memory_export", control) - from src import channels + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") replies = [] assert channels.handle_control_message( @@ -86,3 +96,5 @@ def test_shared_dispatcher_consumes_control_commands(monkeypatch): ) assert replies == ["Export requested"] assert not channels.handle_control_message("hello", "telegram:chat:user", replies.append) + channels.process_control_messages() + assert processed == [True] From e34fc7f76412a832e5259358b1b402c92c505ef7 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 11:09:05 +0300 Subject: [PATCH 11/34] refactor(memory): remove obsolete gateway wrapper Restore direct history and ChromaDB access now that exports run sequentially in the main loop. The gateway lock is no longer needed. --- lib_omegaclaw.metta | 1 - src/helper.py | 4 +- src/memory.metta | 12 ++++-- src/memory_gateway.py | 82 ------------------------------------ tests/test_memory_gateway.py | 33 --------------- 5 files changed, 9 insertions(+), 123 deletions(-) delete mode 100644 src/memory_gateway.py delete mode 100644 tests/test_memory_gateway.py diff --git a/lib_omegaclaw.metta b/lib_omegaclaw.metta index e0dc2884..b8cc0a44 100644 --- a/lib_omegaclaw.metta +++ b/lib_omegaclaw.metta @@ -22,7 +22,6 @@ !(import! &self (library OmegaClaw-Core ./src/fileio.py)) !(import! &self (library OmegaClaw-Core ./src/skills)) !(import! &self (library OmegaClaw-Core ./src/websearch.py)) -!(import! &self (library OmegaClaw-Core ./src/memory_gateway.py)) !(import! &self (library OmegaClaw-Core ./src/memory)) !(import! &self (library OmegaClaw-Core ./src/context)) !(import! &self (library OmegaClaw-Core ./src/loop)) diff --git a/src/helper.py b/src/helper.py index 19742037..6056568f 100644 --- a/src/helper.py +++ b/src/helper.py @@ -9,10 +9,8 @@ try: from src.logger import get_logger - from src.memory_gateway import history_path except ModuleNotFoundError: # running this file directly as a script from logger import get_logger - from memory_gateway import history_path logger = get_logger(__name__) @@ -65,7 +63,7 @@ def extract_timestamp(line): def around_time(needle_time_str, k): needle_time_str = needle_time_str.replace(r'\"', '').replace('"', '').strip() - filename = history_path() + filename = "repos/OmegaClaw-Core/memory/history.metta" target = datetime.strptime(needle_time_str, "%Y-%m-%d %H:%M:%S") best_lineno = None best_line = None diff --git a/src/memory.metta b/src/memory.metta index ecb95a28..39795f29 100644 --- a/src/memory.metta +++ b/src/memory.metta @@ -27,7 +27,11 @@ ""))))) (= (getHistory) - (py-call (memory_gateway.history_tail (maxHistory)))) + (let $history_file + (library OmegaClaw-Core ./memory/history.metta) + (if (exists-file $history_file) ;Safely read the history. Return an empty string if the file does not exist. + (read_file_tail $history_file (maxHistory)) + ""))) (= (addToHistory $lastmessage $response $sexpr $msgnew) (if $msgnew @@ -44,14 +48,14 @@ (py-call (rag.openai_embed (string-safe $str))))) (= (appendToHistory $addition) - (py-call (memory_gateway.append_history (swrite $addition)))) + (append-file-raw (library OmegaClaw-Core ./memory/history.metta) (swrite $addition))) (= (remember $str) - (progn (py-call (memory_gateway.remember $str (embed $str) (get_time_as_string))) + (progn (py-call (lib_chromadb.remember $str (embed $str) (get_time_as_string))) REMEMBER-SUCCESS)) (= (query $str) - (py-call (memory_gateway.query (embed $str) (maxRecallItems)))) + (py-call (lib_chromadb.query (embed $str) (maxRecallItems)))) (= (episodes $time) (py-call (helper.around_time $time (maxEpisodeRecallLines)))) diff --git a/src/memory_gateway.py b/src/memory_gateway.py deleted file mode 100644 index 486ef66b..00000000 --- a/src/memory_gateway.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Thread-safe access to persistent history and vector memory.""" - -import os -import threading -import uuid -from pathlib import Path - -import chromadb - -from src.logger import get_logger - -logger = get_logger(__name__) - -_write_lock = threading.Lock() -_RECORD_KIND = "user_memory" -_client: chromadb.ClientAPI | None = None -_collection = None -_REPO_ROOT = Path(__file__).parent.parent.resolve() - -def memory_dir_path() -> Path: - return Path(os.environ.get("MEMORY_DIR", _REPO_ROOT / "memory")).resolve() - -def history_path() -> Path: - return memory_dir_path() / "history.metta" - -def chroma_db_path() -> Path: - return memory_dir_path() / "chroma_db" - -def _get_collection(): - global _client, _collection - if _collection is None: - db_path = str(chroma_db_path()) - logger.info(f"memory_gateway: opening ChromaDB at {db_path}") - _client = chromadb.PersistentClient(path=db_path) - _collection = _client.get_or_create_collection( - name="memories", embedding_function=None - ) - return _collection - -def append_history(text: str) -> None: - path = history_path() - path.parent.mkdir(parents=True, exist_ok=True) - with _write_lock: - with path.open("a", encoding="utf-8") as history: - history.write(text) - history.write("\n") - -def history_tail(max_chars: int) -> str: - path = history_path() - if not path.exists(): - return "" - with path.open("r", encoding="utf-8", errors="replace") as history: - return history.read()[-max_chars:] - -def remember(content: str, embedding: list[float], time: str) -> str: - item_id = str(uuid.uuid4()) - with _write_lock: - _get_collection().add( - ids=[item_id], - documents=[content], - embeddings=[embedding], - metadatas=[{"time": time, "record_kind": _RECORD_KIND}], - ) - logger.debug(f"memory_gateway: remembered record {item_id}") - return item_id - -def query(query_embedding: list[float], k: int) -> list[list]: - return [[time, content] for _, time, content in query_with_ids(query_embedding, k)] - -def query_with_ids(query_embedding: list[float], k: int) -> list[list]: - result = _get_collection().query( - query_embeddings=[query_embedding], - n_results=k, - include=["documents", "metadatas", "distances"], - ) - ids = result["ids"][0] - documents = result.get("documents", [[]])[0] - metadata = result.get("metadatas", [[]])[0] - return [ - [ids[index], metadata[index].get("time") if metadata[index] else None, documents[index]] - for index in range(len(ids)) - ] diff --git a/tests/test_memory_gateway.py b/tests/test_memory_gateway.py deleted file mode 100644 index 9cea3195..00000000 --- a/tests/test_memory_gateway.py +++ /dev/null @@ -1,33 +0,0 @@ -import importlib -import threading - -import pytest - -EMBEDDING = [0.1, 0.2, 0.3] - -@pytest.fixture(autouse=True) -def isolated_gateway(tmp_path, monkeypatch): - monkeypatch.setenv("MEMORY_DIR", str(tmp_path)) - import src.memory_gateway as gateway - importlib.reload(gateway) - return gateway - -def test_history_append_and_ltm_query(isolated_gateway, tmp_path): - isolated_gateway.append_history("first") - isolated_gateway.append_history("second") - isolated_gateway.remember("portable fact", EMBEDDING, "2026-01-01") - - assert (tmp_path / "history.metta").read_text() == "first\nsecond\n" - assert isolated_gateway.query(EMBEDDING, 1) == [["2026-01-01", "portable fact"]] - -def test_export_lock_blocks_history_write(isolated_gateway, tmp_path): - finished = threading.Event() - thread = threading.Thread( - target=lambda: (isolated_gateway.append_history("concurrent"), finished.set()) - ) - with isolated_gateway._write_lock: - thread.start() - assert not finished.wait(0.05) - thread.join(timeout=2) - assert finished.is_set() - assert (tmp_path / "history.metta").read_text() == "concurrent\n" From 0e79d9ea2a56e3b0548a1bd1902c3d2b0dc6fbdd Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 12:25:25 +0300 Subject: [PATCH 12/34] fix(memory-export): isolate completion delivery failures --- channels/irc.py | 1 - channels/memory_export.py | 9 +++++---- tests/test_memory_export.py | 17 ++++++++++++----- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/channels/irc.py b/channels/irc.py index f924bab5..c1fe0457 100644 --- a/channels/irc.py +++ b/channels/irc.py @@ -100,7 +100,6 @@ def _send_export_reply(nick, text): ): _send(f"PRIVMSG {target} :{line}") - def _flush_outbox(): try: _outbox.flush(_deliver_outbound, _ready_to_send) diff --git a/channels/memory_export.py b/channels/memory_export.py index b72755aa..76fddc9d 100644 --- a/channels/memory_export.py +++ b/channels/memory_export.py @@ -10,7 +10,7 @@ import auth from config import config_get_by_key from src.logger import get_logger -from import_knowledge.memory_portability import MemoryTransfer +from memory_portability import MemoryTransfer logger = get_logger(__name__) @@ -117,8 +117,6 @@ def _handle_confirm(owner_key: str, token: str, deliver_completion) -> str: if not secrets.compare_digest(expected_token, token): return "Invalid token." del _pending_requests[owner_key] - - with _request_lock: _export_requests.append((component, deliver_completion)) return "Export queued. It will run in the next agent iteration." @@ -133,7 +131,10 @@ def process_pending_export() -> None: except Exception as exc: logger.exception(f"memory_export: export failed: {exc}") reply = f"Memory export failed: {exc}" - deliver_completion(reply) + try: + deliver_completion(reply) + except Exception: + logger.exception("memory_export: completion delivery failed") def _format_export(result: dict) -> str: return ( diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index 0d0be87c..adce968b 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -19,12 +19,9 @@ def handler(monkeypatch): logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) monkeypatch.setitem(sys.modules, "src.logger", logger_mod) - import_knowledge = types.ModuleType("import_knowledge") - import_knowledge.__path__ = [] - monkeypatch.setitem(sys.modules, "import_knowledge", import_knowledge) - mp_mod = types.ModuleType("import_knowledge.memory_portability") + mp_mod = types.ModuleType("memory_portability") mp_mod.MemoryTransfer = object - monkeypatch.setitem(sys.modules, "import_knowledge.memory_portability", mp_mod) + monkeypatch.setitem(sys.modules, "memory_portability", mp_mod) spec = importlib.util.spec_from_file_location( "memory_export_under_test", @@ -79,6 +76,16 @@ def test_export_completion_is_delivered_after_loop_processing(handler): assert exported == ["both"] assert "memory.tar.gz" in delivered[-1] +def test_failed_completion_delivery_does_not_interrupt_loop(handler): + handler._get_transfer = lambda: types.SimpleNamespace(export=lambda _component: {}) + token = handler.handle_export_command("/memory-export history").split()[-1] + + def fail_delivery(_message): + raise RuntimeError("send failed") + + handler.handle_export_command(f"/memory-export confirm {token}", deliver_completion=fail_delivery) + handler.process_pending_export() + def test_shared_dispatcher_consumes_control_commands(monkeypatch): control = types.ModuleType("memory_export") control.is_export_command = lambda text: text == "/memory-export both" From d8bb83eaf31832b1865740fee873004b84bd0b0f Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 12:26:57 +0300 Subject: [PATCH 13/34] build(memory): Install the tagged v0.2.1 standalone package --- Dockerfile | 8 ++++---- entrypoint.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index b6736f73..6cfe62ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,11 +69,11 @@ SentenceTransformer(model_name) print("Model download complete.") PY -ARG IMPORT_KNOWLEDGE_REPO=https://github.com/Bereket-Eshete/import-knowledge-package.git -ARG IMPORT_KNOWLEDGE_REF=v0.2.1 -RUN test -n "${IMPORT_KNOWLEDGE_REF}" \ +ARG MEMORY_PORTABILITY_REPO=https://github.com/Bereket-Eshete/memory-portability-package.git +ARG MEMORY_PORTABILITY_REF=v0.2.1 +RUN test -n "${MEMORY_PORTABILITY_REF}" \ && python3 -m pip install --no-cache-dir --break-system-packages \ - "git+${IMPORT_KNOWLEDGE_REPO}@${IMPORT_KNOWLEDGE_REF}" + "git+${MEMORY_PORTABILITY_REPO}@${MEMORY_PORTABILITY_REF}" FROM builder AS versioned-source diff --git a/entrypoint.sh b/entrypoint.sh index d10733f3..de005524 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -50,7 +50,7 @@ if [[ "${MEMORY_TRANSFER_MOUNTED:-0}" == "1" ]]; then fi # Recover an interrupted import before starting the agent. -su nobody -s /bin/sh -c 'cd "$1" && exec python3 -m import_knowledge.memory_portability recover' \ +su nobody -s /bin/sh -c 'cd "$1" && exec python3 -m memory_portability recover' \ sh "$OMEGACLAW_DIR" \ || { echo "Memory import recovery failed. Aborting startup." >&2; exit 1; } @@ -69,7 +69,7 @@ if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then [[ "${MEMORY_IMPORT_NO_VECTOR:-0}" == "1" ]] && import_args+=(--no-vector) [[ "${MEMORY_IMPORT_ONLY_HISTORY:-0}" == "1" ]] && import_args+=(--no-vector) echo "memory_portability: importing ${MEMORY_IMPORT_FILE}" - su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m import_knowledge.memory_portability import "$@"' \ + su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m memory_portability import "$@"' \ -- sh "$OMEGACLAW_DIR" "${import_args[@]}" \ || { echo "Memory import failed. Aborting startup." >&2; exit 1; } echo "memory_portability: import complete" From e81329d2960f1e7e81c7c3e3c24bc6fa77a1a89f Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 19:12:02 +0300 Subject: [PATCH 14/34] feat: integrate memory portability package --- Dockerfile | 6 ----- channels/memory_export.py | 37 +++++++------------------------ channels/wschat.py | 3 ++- entrypoint.sh | 46 ++++++++++++++------------------------- requirements.txt | 2 +- scripts/omegaclaw | 37 ++++++++++++++----------------- src/channels.py | 14 +++--------- src/loop.metta | 1 - 8 files changed, 46 insertions(+), 100 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6cfe62ca..d43eb119 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,12 +69,6 @@ SentenceTransformer(model_name) print("Model download complete.") PY -ARG MEMORY_PORTABILITY_REPO=https://github.com/Bereket-Eshete/memory-portability-package.git -ARG MEMORY_PORTABILITY_REF=v0.2.1 -RUN test -n "${MEMORY_PORTABILITY_REF}" \ - && python3 -m pip install --no-cache-dir --break-system-packages \ - "git+${MEMORY_PORTABILITY_REPO}@${MEMORY_PORTABILITY_REF}" - FROM builder AS versioned-source WORKDIR /omegaclaw-source diff --git a/channels/memory_export.py b/channels/memory_export.py index 76fddc9d..20546ea3 100644 --- a/channels/memory_export.py +++ b/channels/memory_export.py @@ -1,20 +1,17 @@ -"""Shared authenticated /memory-export command handling.""" +"""Shared /memory-export command handling.""" -from collections import deque -import os import secrets import threading import time from pathlib import Path -import auth from config import config_get_by_key from src.logger import get_logger from memory_portability import MemoryTransfer logger = get_logger(__name__) -_TRANSFER_DIR = Path(os.environ.get("MEMORY_TRANSFER_DIR", "/memory-transfer")) +_TRANSFER_DIR = Path("/memory-transfer") _transfer = None @@ -28,14 +25,11 @@ def _get_transfer() -> MemoryTransfer: _request_lock = threading.Lock() _pending_requests: dict[str, tuple[str, str, float]] = {} -_export_requests = deque() _VALID_COMPONENTS = ("history", "ltm", "both") def is_export_enabled() -> bool: - value = os.environ.get("OMEGACLAW_memoryExportEnabled") - if value is None: - value = config_get_by_key("memoryExportEnabled", False) + value = config_get_by_key("memoryExportEnabled", False) return value is True or ( isinstance(value, str) and value.strip().lower() == "true" ) @@ -63,15 +57,12 @@ def _issue_token(owner_key: str, component: str) -> str: def handle_export_command( text: str, owner_key: str = "default-owner", - deliver_completion=lambda _message: None, ) -> str | None: stripped = text.strip() if not is_export_command(stripped): return None - if not auth.is_auth_enabled(): - return None if not is_export_enabled(): return None @@ -84,7 +75,7 @@ def handle_export_command( return _handle_request(owner_key, sub) if sub == "confirm": - return _handle_confirm(owner_key, arg, deliver_completion) + return _handle_confirm(owner_key, arg) return ( "Unknown /memory-export command. " @@ -102,7 +93,7 @@ def _handle_request(owner_key: str, component: str) -> str: f"/memory-export confirm {token}" ) -def _handle_confirm(owner_key: str, token: str, deliver_completion) -> str: +def _handle_confirm(owner_key: str, token: str) -> str: if not token: return "Usage: /memory-export confirm " @@ -117,30 +108,18 @@ def _handle_confirm(owner_key: str, token: str, deliver_completion) -> str: if not secrets.compare_digest(expected_token, token): return "Invalid token." del _pending_requests[owner_key] - _export_requests.append((component, deliver_completion)) - return "Export queued. It will run in the next agent iteration." - -def process_pending_export() -> None: - with _request_lock: - if not _export_requests: - return - component, deliver_completion = _export_requests.popleft() try: result = _get_transfer().export(component) - reply = _format_export(result) + return _format_export(result) except Exception as exc: logger.exception(f"memory_export: export failed: {exc}") - reply = f"Memory export failed: {exc}" - try: - deliver_completion(reply) - except Exception: - logger.exception("memory_export: completion delivery failed") + return f"Memory export failed: {exc}" def _format_export(result: dict) -> str: return ( "Memory export complete\n" f"File: {result.get('filename')}\n" f"Size: {result.get('size')} bytes\n" - f"SHA-256: {result.get('checksum')}\n" + f"SHA-256: {result.get('sha256', result.get('checksum'))}\n" f"Records: {result.get('record_count')}" ) diff --git a/channels/wschat.py b/channels/wschat.py index 77d9fff1..52dbacc6 100644 --- a/channels/wschat.py +++ b/channels/wschat.py @@ -70,6 +70,7 @@ sys.path.insert(0, str(_REPO_ROOT)) from src.logger import get_logger +from memory_export import is_export_command try: import channels except ModuleNotFoundError: @@ -194,7 +195,7 @@ def _handle_frame(raw_message): if not isinstance(seq, int) or not isinstance(text, str): logger.warning(f"Ignoring malformed user_message frame: {frame!r}") return - if channels.is_control_command(text): + if is_export_command(text): logger.info("Ignoring unavailable memory-export command on WebSocket channel") return _enqueue_user_message(seq, text) diff --git a/entrypoint.sh b/entrypoint.sh index de005524..d47068fd 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -40,38 +40,26 @@ if [[ "${IMPORT_KB_ON_START}" == "1" ]]; then su nobody -s /bin/sh -c "${OMEGACLAW_DIR}/scripts/import_knowledge.sh" fi -MEMORY_TRANSFER_DIR="${MEMORY_TRANSFER_DIR:-/memory-transfer}" -export MEMORY_TRANSFER_DIR - -# Verify that the agent user can write the mounted transfer directory. -if [[ "${MEMORY_TRANSFER_MOUNTED:-0}" == "1" ]]; then - su nobody -s /bin/sh -c 'test -d "$1" && test -w "$1"' sh "$MEMORY_TRANSFER_DIR" \ - || { echo "Memory transfer directory is not writable by the agent user." >&2; exit 1; } -fi - -# Recover an interrupted import before starting the agent. -su nobody -s /bin/sh -c 'cd "$1" && exec python3 -m memory_portability recover' \ - sh "$OMEGACLAW_DIR" \ +MEMORY_PORTABILITY_PYTHON='from memory_portability import MemoryTransfer; MemoryTransfer().recover()' +export MEMORY_PORTABILITY_PYTHON +su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \ || { echo "Memory import recovery failed. Aborting startup." >&2; exit 1; } +unset MEMORY_PORTABILITY_PYTHON -# Validate the archive filename again at the container boundary. if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then - if [[ ! "${MEMORY_IMPORT_FILE}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.tar\.gz$ ]]; then - echo "MEMORY_IMPORT_FILE must be a plain filename, not a path: ${MEMORY_IMPORT_FILE}" >&2 - exit 1 - fi - case "${MEMORY_IMPORT_MODE:-overwrite}" in - overwrite|append) ;; - *) echo "MEMORY_IMPORT_MODE must be overwrite or append" >&2; exit 1 ;; - esac - import_args=(--transfer-dir "$MEMORY_TRANSFER_DIR" --filename "${MEMORY_IMPORT_FILE}" --mode "${MEMORY_IMPORT_MODE:-overwrite}") - [[ "${MEMORY_IMPORT_NO_HISTORY:-0}" == "1" ]] && import_args+=(--no-history) - [[ "${MEMORY_IMPORT_NO_VECTOR:-0}" == "1" ]] && import_args+=(--no-vector) - [[ "${MEMORY_IMPORT_ONLY_HISTORY:-0}" == "1" ]] && import_args+=(--no-vector) echo "memory_portability: importing ${MEMORY_IMPORT_FILE}" - su nobody -s /bin/sh -c 'cd "$1" && shift && exec python3 -m memory_portability import "$@"' \ - -- sh "$OMEGACLAW_DIR" "${import_args[@]}" \ + MEMORY_PORTABILITY_PYTHON='import os +from memory_portability import MemoryTransfer +MemoryTransfer().import_archive( + os.environ["MEMORY_IMPORT_FILE"], + mode=os.environ.get("MEMORY_IMPORT_MODE", "overwrite"), + include_history=os.environ.get("MEMORY_IMPORT_NO_HISTORY") != "1", + include_vectors=os.environ.get("MEMORY_IMPORT_NO_VECTOR") != "1", +)' + export MEMORY_PORTABILITY_PYTHON + su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \ || { echo "Memory import failed. Aborting startup." >&2; exit 1; } + unset MEMORY_PORTABILITY_PYTHON echo "memory_portability: import complete" fi @@ -79,9 +67,7 @@ fi SAFE_VARS="HOME USER PATH HOSTNAME TERM LANG LC_ALL \ PYTHONDONTWRITEBYTECODE PYTHONUNBUFFERED \ HF_HOME SENTENCE_TRANSFORMERS_HOME HF_HUB_OFFLINE TRANSFORMERS_OFFLINE \ - EMBEDDING_PROVIDER \ - OMEGACLAW_memoryExportEnabled \ - OMEGACLAW_DIR MEMORY_DIR MEMORY_TRANSFER_DIR TEST_SERVER_IP" + OMEGACLAW_DIR MEMORY_DIR TEST_SERVER_IP" env_args="" for var in $SAFE_VARS; do diff --git a/requirements.txt b/requirements.txt index 785be39a..ff1cc221 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ chromadb==1.5.9 openai==2.38.0 transformers==5.8.0 sentence-transformers==5.5.1 -import-kb==0.1.8 +import-kb==0.1.9 py-landlock==0.1.1 pyyaml==6.0.3 ddgs==9.14.4 diff --git a/scripts/omegaclaw b/scripts/omegaclaw index 570b538a..d00a89ab 100755 --- a/scripts/omegaclaw +++ b/scripts/omegaclaw @@ -309,7 +309,7 @@ def _choose_memory_transfer(): continue while True: - enable = input("Enable authenticated memory export for this instance? [y/N]: ").strip().lower() + enable = input("Enable memory export for this instance? [y/N]: ").strip().lower() if enable in ("", "n", "no"): return path, "0" if enable in ("y", "yes"): @@ -540,7 +540,7 @@ help() { echo echo -e "Memory portability options:" echo -e "\t--memory-transfer-dir mount host directory for memory export/import archives" - echo -e "\t--enable-memory-export enable authenticated memory export" + echo -e "\t--enable-memory-export enable memory export" echo -e "\t--memory-import restore archive from transfer directory before startup" echo -e "\t--memory-mode overwrite|append import mode (default: overwrite)" echo -e "\t--only-history restore history only" @@ -576,7 +576,6 @@ options() { memory_import_mode="overwrite" memory_import_no_history=0 memory_import_no_vector=0 - memory_import_only_history=0 memory_export_enabled=0 if [[ "$#" -eq 0 ]]; then @@ -637,7 +636,7 @@ options() { shift ;; --only-history) - memory_import_only_history=1 + memory_import_no_vector=1 shift ;; --version|-v) version; return 0;; @@ -725,15 +724,13 @@ options() { return 1 fi - if [[ "${memory_import_no_history}" == "1" && - ( "${memory_import_no_vector}" == "1" || "${memory_import_only_history}" == "1" ) ]]; then + if [[ "${memory_import_no_history}" == "1" && "${memory_import_no_vector}" == "1" ]]; then echo "--no-history cannot be combined with --no-vector or --only-history" >&2 return 1 fi if [[ -z "${memory_import_file}" && ( "${memory_import_no_history}" == "1" || - "${memory_import_no_vector}" == "1" || - "${memory_import_only_history}" == "1" ) ]]; then + "${memory_import_no_vector}" == "1" ) ]]; then echo "Import component flags require --memory-import" >&2 return 1 fi @@ -745,15 +742,13 @@ options() { } start() { - docker rm -f omegaclaw 2>/dev/null || true docker pull "${image}" 2>/dev/null || true container_log_config_path="" log_config_volume=() memory_transfer_volume=() memory_import_env=() - memory_transfer_env=() - memory_export_env=() + memory_export_option=() if [ -n "${log_config_path}" ]; then if [ ! -f "${log_config_path}" ]; then @@ -768,14 +763,16 @@ start() { if [ -n "${memory_transfer_dir}" ]; then memory_transfer_volume=(--volume "${memory_transfer_dir}:/memory-transfer") - memory_transfer_env=( - -e MEMORY_TRANSFER_MOUNTED=1 - -e MEMORY_TRANSFER_DIR=/memory-transfer - ) + if ! docker run --rm --user 65534:65534 --entrypoint /bin/sh \ + --volume "${memory_transfer_dir}:/memory-transfer" "${image}" \ + -c 'test -d /memory-transfer && test -w /memory-transfer'; then + echo "--memory-transfer-dir must be writable by container user 65534:65534" >&2 + return 1 + fi fi if [[ "${memory_export_enabled}" == "1" ]]; then - memory_export_env=(-e OMEGACLAW_memoryExportEnabled=true) + memory_export_option=("memoryExportEnabled=true") fi if [ -n "${memory_import_file}" ]; then @@ -789,11 +786,10 @@ start() { if [[ "${memory_import_no_vector}" == "1" ]]; then memory_import_env+=(-e MEMORY_IMPORT_NO_VECTOR=1) fi - if [[ "${memory_import_only_history}" == "1" ]]; then - memory_import_env+=(-e MEMORY_IMPORT_ONLY_HISTORY=1) - fi fi + docker rm -f omegaclaw 2>/dev/null || true + docker_cmd=( docker run -d -it --name omegaclaw @@ -812,8 +808,6 @@ start() { -e "OMEGACLAW_OPENCLAW_TOKEN=${OMEGACLAW_OPENCLAW_TOKEN:-}" -e OMEGACLAW_AUTH_SECRET="$OMEGACLAW_AUTH_SECRET" -e IMPORT_KB_ON_START="${IMPORT_KB_ON_START:-0}" - ${memory_transfer_env[@]+"${memory_transfer_env[@]}"} - ${memory_export_env[@]+"${memory_export_env[@]}"} ${memory_import_env[@]+"${memory_import_env[@]}"} "$image" "commchannel=${commchannel}" @@ -821,6 +815,7 @@ start() { "embeddingprovider=${embeddingprovider}" "securityPolicyPath=/PeTTa/repos/OmegaClaw-Core/profile/policy.yaml" "memoryDirectory=\$MEMORY_DIR" + ${memory_export_option[@]+"${memory_export_option[@]}"} ) if [ -n "${container_log_config_path}" ]; then diff --git a/src/channels.py b/src/channels.py index e0becd0a..90e2b52d 100644 --- a/src/channels.py +++ b/src/channels.py @@ -4,24 +4,16 @@ _commChannelRegistry = {} -def is_control_command(text: str) -> bool: - from memory_export import is_export_command - return is_export_command(text) - def handle_control_message(text: str, owner_key: str, deliver_reply) -> bool: - from memory_export import handle_export_command + from memory_export import handle_export_command, is_export_command - if not is_control_command(text): + if not is_export_command(text): return False - reply = handle_export_command(text, owner_key, deliver_reply) + reply = handle_export_command(text, owner_key) if reply is not None: deliver_reply(reply) return True -def process_control_messages() -> None: - from memory_export import process_pending_export - process_pending_export() - class CommChannel: """Communication channel implementation""" diff --git a/src/loop.metta b/src/loop.metta index 3db13bd8..eced53cd 100644 --- a/src/loop.metta +++ b/src/loop.metta @@ -68,7 +68,6 @@ (commChannelSend (version)) (llmProviderStart (provider))) (change-state! &loops (- (get-state &loops) 1))) - (py-call (channels.process_control_messages)) (let $prompt (getContext) (progn (log INFO "loop" (---------iteration $k)) (heartbeat $k) From 69ac2c87029c9c5ae9ecebef8e935231efe02622 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 19:12:13 +0300 Subject: [PATCH 15/34] test: cover synchronous memory export handling --- tests/test_memory_export.py | 64 ++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index adce968b..abc64b02 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -1,5 +1,6 @@ import importlib import importlib.util +import json import sys import time import types @@ -32,10 +33,8 @@ def handler(monkeypatch): module.is_export_enabled = lambda: True return module -def test_export_command_requires_auth_and_policy(handler): - handler.auth.is_auth_enabled = lambda: False - assert handler.handle_export_command("/memory-export both") is None - handler.auth.is_auth_enabled = lambda: True +def test_export_command_requires_policy_but_not_auth(handler): + assert "Export requested" in handler.handle_export_command("/memory-export both") handler.is_export_enabled = lambda: False assert handler.handle_export_command("/memory-export both") is None @@ -53,45 +52,54 @@ def test_expired_and_other_owner_tokens_cannot_start_export(handler): f"/memory-export confirm {token}", "owner-a" ).lower() -def test_export_completion_is_delivered_after_loop_processing(handler): - delivered = [] +def test_confirmation_exports_immediately(handler): exported = [] handler._get_transfer = lambda: types.SimpleNamespace( export=lambda component: exported.append(component) or { "filename": "memory.tar.gz", "size": 1, - "checksum": "abc", + "sha256": "abc", "record_count": 1, } ) - token = handler.handle_export_command("/memory-export both", "owner-a", delivered.append).split()[-1] - delivered.append( - handler.handle_export_command( - f"/memory-export confirm {token}", "owner-a", delivered.append - ) - ) - assert delivered == ["Export queued. It will run in the next agent iteration."] - assert exported == [] - handler.process_pending_export() + token = handler.handle_export_command("/memory-export both", "owner-a").split()[-1] + reply = handler.handle_export_command(f"/memory-export confirm {token}", "owner-a") assert exported == ["both"] - assert "memory.tar.gz" in delivered[-1] + assert "memory.tar.gz" in reply + assert "SHA-256: abc" in reply -def test_failed_completion_delivery_does_not_interrupt_loop(handler): - handler._get_transfer = lambda: types.SimpleNamespace(export=lambda _component: {}) - token = handler.handle_export_command("/memory-export history").split()[-1] +def test_websocket_ignores_memory_export(monkeypatch): + config = types.ModuleType("config") + config.config_get_by_key = lambda key, default=None: default + logger_mod = types.ModuleType("src.logger") + logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) + control = types.ModuleType("memory_export") + control.is_export_command = lambda text: text.startswith("/memory-export") + channels = types.ModuleType("channels") + channels.CommChannel = object + channels.registerCommChannel = lambda *args: None + monkeypatch.setitem(sys.modules, "config", config) + monkeypatch.setitem(sys.modules, "src.logger", logger_mod) + monkeypatch.setitem(sys.modules, "memory_export", control) + monkeypatch.setitem(sys.modules, "channels", channels) - def fail_delivery(_message): - raise RuntimeError("send failed") + spec = importlib.util.spec_from_file_location( + "wschat_under_test", REPO_ROOT / "channels" / "wschat.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + received = [] + module._enqueue_user_message = lambda *args: received.append(args) - handler.handle_export_command(f"/memory-export confirm {token}", deliver_completion=fail_delivery) - handler.process_pending_export() + module._handle_frame(json.dumps({ + "type": "user_message", "seq": 1, "text": "/memory-export both" + })) + assert received == [] def test_shared_dispatcher_consumes_control_commands(monkeypatch): control = types.ModuleType("memory_export") control.is_export_command = lambda text: text == "/memory-export both" - control.handle_export_command = lambda text, owner, deliver: "Export requested" - processed = [] - control.process_pending_export = lambda: processed.append(True) + control.handle_export_command = lambda text, owner: "Export requested" monkeypatch.setitem(sys.modules, "memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) @@ -103,5 +111,3 @@ def test_shared_dispatcher_consumes_control_commands(monkeypatch): ) assert replies == ["Export requested"] assert not channels.handle_control_message("hello", "telegram:chat:user", replies.append) - channels.process_control_messages() - assert processed == [True] From 26819eeef67676a80c985a58e8df10af52ef0cb3 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Thu, 20 Aug 2026 19:12:23 +0300 Subject: [PATCH 16/34] docs: clarify memory portability operations --- docs/reference-memory-portability.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/reference-memory-portability.md b/docs/reference-memory-portability.md index 0fb025fb..418d02da 100644 --- a/docs/reference-memory-portability.md +++ b/docs/reference-memory-portability.md @@ -21,8 +21,9 @@ The transfer directory must be writable by the container's agent user. ## Export -In a private, authenticated supported chat, request one component and confirm -the returned short-lived token: +In a supported chat, request one component and confirm the returned short-lived +token. When channel authentication is active, only the authenticated owner can +use these commands. When it is disabled, normal channel access rules apply: ```text /memory-export history @@ -31,9 +32,9 @@ the returned short-lived token: /memory-export confirm ``` -The confirmed export runs in the next agent iteration. Completion is delivered -only to the owner who started it and includes the filename, record count, size, -and SHA-256. +The confirmed export runs immediately in the channel harness. Completion is +delivered to the requester and includes the filename, record count, size, and +SHA-256. Archives contain selected persistent user memory only: From d4d412379bb52da7ec2f56f4457b1902e060e464 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Fri, 21 Aug 2026 14:20:20 +0300 Subject: [PATCH 17/34] fix: notify WebSocket users about unsupported memory export --- channels/wschat.py | 2 +- tests/test_memory_export.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/channels/wschat.py b/channels/wschat.py index 52dbacc6..e90c848f 100644 --- a/channels/wschat.py +++ b/channels/wschat.py @@ -196,7 +196,7 @@ def _handle_frame(raw_message): logger.warning(f"Ignoring malformed user_message frame: {frame!r}") return if is_export_command(text): - logger.info("Ignoring unavailable memory-export command on WebSocket channel") + send_message("Memory export is not supported on the WebSocket channel.") return _enqueue_user_message(seq, text) return diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index abc64b02..0cb434d3 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -89,12 +89,15 @@ def test_websocket_ignores_memory_export(monkeypatch): module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) received = [] + replies = [] module._enqueue_user_message = lambda *args: received.append(args) + module.send_message = replies.append module._handle_frame(json.dumps({ "type": "user_message", "seq": 1, "text": "/memory-export both" })) assert received == [] + assert replies == ["Memory export is not supported on the WebSocket channel."] def test_shared_dispatcher_consumes_control_commands(monkeypatch): control = types.ModuleType("memory_export") From 5cdc8b9c4c73193af0d6b7e58d5eb86687382408 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Fri, 21 Aug 2026 14:20:32 +0300 Subject: [PATCH 18/34] fix: preserve embedding provider for memory export --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index d47068fd..09e8c3b5 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -67,7 +67,7 @@ fi SAFE_VARS="HOME USER PATH HOSTNAME TERM LANG LC_ALL \ PYTHONDONTWRITEBYTECODE PYTHONUNBUFFERED \ HF_HOME SENTENCE_TRANSFORMERS_HOME HF_HUB_OFFLINE TRANSFORMERS_OFFLINE \ - OMEGACLAW_DIR MEMORY_DIR TEST_SERVER_IP" + EMBEDDING_PROVIDER OMEGACLAW_DIR MEMORY_DIR TEST_SERVER_IP" env_args="" for var in $SAFE_VARS; do From 88b2b3cbae19e07ee30718db58b78f8136a8e49a Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Fri, 21 Aug 2026 14:20:45 +0300 Subject: [PATCH 19/34] fix: create missing memory transfer directories --- docs/reference-memory-portability.md | 6 +++--- scripts/omegaclaw | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference-memory-portability.md b/docs/reference-memory-portability.md index 418d02da..4d44f6a1 100644 --- a/docs/reference-memory-portability.md +++ b/docs/reference-memory-portability.md @@ -6,9 +6,9 @@ workflow, not an LLM skill. ## Setup -Choose an absolute host directory for archives. The launcher mounts it at the -fixed container path `/memory-transfer`; the agent never accepts arbitrary -runtime export paths. +Choose an absolute host directory for archives. The launcher creates it when +needed and mounts it at the fixed container path `/memory-transfer`; the agent +never accepts arbitrary runtime export paths. ```sh scripts/omegaclaw start -p OpenAI -t telegram \ diff --git a/scripts/omegaclaw b/scripts/omegaclaw index d00a89ab..d9689906 100755 --- a/scripts/omegaclaw +++ b/scripts/omegaclaw @@ -694,8 +694,8 @@ options() { echo "--memory-transfer-dir must be an absolute path: ${memory_transfer_dir}" >&2 return 1 fi - if [[ ! -d "${memory_transfer_dir}" ]]; then - echo "--memory-transfer-dir does not exist: ${memory_transfer_dir}" >&2 + if ! mkdir -p "${memory_transfer_dir}"; then + echo "Could not create --memory-transfer-dir: ${memory_transfer_dir}" >&2 return 1 fi if [[ ! -w "${memory_transfer_dir}" ]]; then From 7e402ad0791d0ac83d963c0c3d0ac245e8168816 Mon Sep 17 00:00:00 2001 From: Bereket-Eshete Date: Fri, 21 Aug 2026 14:20:54 +0300 Subject: [PATCH 20/34] docs: remove duplicate python bridges reference --- docs/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/README.md b/docs/README.md index 0afd95fe..8ba82079 100644 --- a/docs/README.md +++ b/docs/README.md @@ -58,7 +58,6 @@ User-facing MeTTa skills the agent invokes. Each page follows the template **Sig - [reference-channels.md](./reference-channels.md) — IRC, Telegram, Slack, Mattermost, WebSocket, and websearch adapters plus the channel contract - [reference-python-bridges.md](./reference-python-bridges.md) — `lib_llm_ext.py`, `src/helper.py`, `src/skills.pl` - [reference-memory-portability.md](./reference-memory-portability.md) — Operator backup, restore, and archive-transfer workflow -- [reference-python-bridges.md](./reference-python-bridges.md) — `lib_llm_ext.py`, `src/agentverse.py`, `src/helper.py`, `src/skills.pl` ### Internals From b6ef8215e0b9ee82a3462309e2f684c7b78dc036 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Fri, 21 Aug 2026 15:34:08 +0300 Subject: [PATCH 21/34] Fix: embedding provider now visible to memory_portability package and resolved bootstrap conflict --- channels/memory_export.py | 6 ++++++ scripts/omegaclaw | 18 ++++++++++++++---- tests/test_memory_export.py | 22 ++++++++++++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/channels/memory_export.py b/channels/memory_export.py index 20546ea3..4b3d6c21 100644 --- a/channels/memory_export.py +++ b/channels/memory_export.py @@ -1,5 +1,6 @@ """Shared /memory-export command handling.""" +import os import secrets import threading import time @@ -18,6 +19,11 @@ def _get_transfer() -> MemoryTransfer: global _transfer if _transfer is None: + embedding_provider = str(config_get_by_key("embeddingprovider", "Local")).strip() + if embedding_provider.casefold() not in {"local", "openai"}: + raise ValueError(f"Unsupported embedding provider: {embedding_provider!r}") + + os.environ["EMBEDDING_PROVIDER"] = embedding_provider _transfer = MemoryTransfer(_TRANSFER_DIR) return _transfer diff --git a/scripts/omegaclaw b/scripts/omegaclaw index d9689906..9f684565 100755 --- a/scripts/omegaclaw +++ b/scripts/omegaclaw @@ -337,7 +337,11 @@ def _write_kv(fp, key, value): fp.write(f"{key}={shlex.quote(str(value))}\n") -def config_run_omegaclaw(config_output_path): +def config_run_omegaclaw( + config_output_path, + configured_memory_transfer_dir="", + configured_memory_export_enabled="0", +): print(" ") print("Welcome to OmegaClaw!") print(" ") @@ -346,7 +350,12 @@ def config_run_omegaclaw(config_output_path): channel_config = _choose_channel() provider, embeddingprovider, api_token_var, model, openaiapi_url, token = _choose_provider() import_kb_on_start = _choose_import_kb() - memory_transfer_dir, memory_export_enabled = _choose_memory_transfer() + if configured_memory_transfer_dir: + memory_transfer_dir = configured_memory_transfer_dir + memory_export_enabled = configured_memory_export_enabled + print(f"Using memory transfer directory: {memory_transfer_dir}") + else: + memory_transfer_dir, memory_export_enabled = _choose_memory_transfer() with open(config_output_path, "w", encoding="utf-8") as f: _write_kv(f, "api_token_var", api_token_var) @@ -364,10 +373,11 @@ def config_run_omegaclaw(config_output_path): if __name__ == "__main__": - config_run_omegaclaw(sys.argv[1]) + config_run_omegaclaw(*sys.argv[1:4]) PY -python3 "$tmp_py_file" "$tmp_config_file" Date: Fri, 21 Aug 2026 17:28:19 +0300 Subject: [PATCH 22/34] Fix: lazily load memory portability for export --- channels/memory_export.py | 10 +++++++--- tests/test_memory_export.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/channels/memory_export.py b/channels/memory_export.py index 4b3d6c21..5eb8eda9 100644 --- a/channels/memory_export.py +++ b/channels/memory_export.py @@ -8,7 +8,6 @@ from config import config_get_by_key from src.logger import get_logger -from memory_portability import MemoryTransfer logger = get_logger(__name__) @@ -16,13 +15,18 @@ _transfer = None -def _get_transfer() -> MemoryTransfer: +def _get_transfer(): global _transfer if _transfer is None: + # Keep the runtime package optional while channel modules are imported. + # The Docker image installs it, but lightweight CI/unit environments do + # not need it unless an export is actually executed. + from memory_portability import MemoryTransfer + embedding_provider = str(config_get_by_key("embeddingprovider", "Local")).strip() if embedding_provider.casefold() not in {"local", "openai"}: raise ValueError(f"Unsupported embedding provider: {embedding_provider!r}") - + os.environ["EMBEDDING_PROVIDER"] = embedding_provider _transfer = MemoryTransfer(_TRANSFER_DIR) return _transfer diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index 5e801c35..f9e64842 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -21,9 +21,7 @@ def handler(monkeypatch): logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) monkeypatch.setitem(sys.modules, "src.logger", logger_mod) - mp_mod = types.ModuleType("memory_portability") - mp_mod.MemoryTransfer = object - monkeypatch.setitem(sys.modules, "memory_portability", mp_mod) + monkeypatch.delitem(sys.modules, "memory_portability", raising=False) spec = importlib.util.spec_from_file_location( "memory_export_under_test", @@ -39,6 +37,10 @@ def test_export_command_requires_policy_but_not_auth(handler): handler.is_export_enabled = lambda: False assert handler.handle_export_command("/memory-export both") is None +def test_module_import_does_not_require_memory_portability(handler): + assert "memory_portability" not in sys.modules + assert handler.is_export_command("/memory-export both") + def test_expired_and_other_owner_tokens_cannot_start_export(handler): token = handler.handle_export_command("/memory-export history", "owner-a").split()[-1] assert "Invalid token" in handler.handle_export_command( @@ -77,7 +79,9 @@ def __init__(self, transfer_dir): created.append((transfer_dir, os.environ["EMBEDDING_PROVIDER"])) monkeypatch.delenv("EMBEDDING_PROVIDER", raising=False) - monkeypatch.setattr(handler, "MemoryTransfer", FakeTransfer) + mp_mod = types.ModuleType("memory_portability") + mp_mod.MemoryTransfer = FakeTransfer + monkeypatch.setitem(sys.modules, "memory_portability", mp_mod) monkeypatch.setattr( handler, "config_get_by_key", From e830bd99817fd32fc99300a27ae23f6ef7fb7aec Mon Sep 17 00:00:00 2001 From: CodersKin Date: Sat, 22 Aug 2026 23:04:08 +0300 Subject: [PATCH 23/34] chore: updated README to include information regarding import memory being unsupported in the standalone run of omegaclaw --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d11eb869..eda541e7 100644 --- a/README.md +++ b/README.md @@ -100,8 +100,15 @@ docker volume rm omegaclaw-memory ### Memory portability Memory export is disabled by default. See the [memory portability reference](./docs/reference-memory-portability.md) -for setup, export controls, archive contents, and import modes. To restore an -archive while upgrading to a tagged image, use the same transfer directory: +for setup, export controls, archive contents, and import modes. + +> **Current limitation:** Memory import and interrupted-import recovery are +> supported only when OmegaClaw is run through Docker using +> `scripts/omegaclaw`. Standalone execution does not invoke the container +> entrypoint and therefore does not support memory import or recovery in this +> release. + +To restore an archive while upgrading to a tagged image, use the same transfer directory: ```sh scripts/omegaclaw start -d singularitynet/omegaclaw: -p OpenAI -t telegram \ From a5e47d28b340a645477b08b8fc84c3e4486ea238 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Sun, 23 Aug 2026 00:22:48 +0300 Subject: [PATCH 24/34] Fix: moved export handler to commChannelReceive in channels.py. --- channels/irc.py | 18 +----- channels/mattermost.py | 29 +-------- channels/memory_export.py | 3 - channels/slack.py | 76 ++++++++++------------ channels/telegram.py | 15 +---- channels/wschat.py | 4 -- src/channels.py | 37 +++++++++-- tests/test_memory_export.py | 121 ++++++++++++++++++++++++++++++++---- 8 files changed, 176 insertions(+), 127 deletions(-) diff --git a/channels/irc.py b/channels/irc.py index c1fe0457..82a245ba 100644 --- a/channels/irc.py +++ b/channels/irc.py @@ -87,19 +87,6 @@ def _deliver_outbound(chunk): _send(f"PRIVMSG {_channel} :{chunk}") -def _send_export_reply(nick, text): - """Send export-control output privately to the authenticated requester.""" - target = _normalize_nick(nick) - if not target or any(char.isspace() for char in target): - raise ValueError("Invalid IRC nickname for export response") - for line in textwrap.wrap( - str(text).replace("\r", "").replace("\n", " "), - width=400, - break_long_words=True, - break_on_hyphens=False, - ): - _send(f"PRIVMSG {target} :{line}") - def _flush_outbox(): try: _outbox.flush(_deliver_outbound, _ready_to_send) @@ -168,10 +155,7 @@ def _irc_session(channel, server, port, nick): msg = trailing.split(" :", 1)[1] state = _is_allowed_message(sender_nick, msg) if state == "allow": - owner_key = f"irc:{_normalize_nick(sender_nick)}" - deliver_reply = lambda message, target=sender_nick: _send_export_reply(target, message) - if not channels.handle_control_message(msg, owner_key, deliver_reply): - _set_last(f"{sender_nick}: {msg}") + _set_last(f"{sender_nick}: {msg}") elif state == "auth_bound": send_message(f"Authentication successful for {sender_nick}.") except Exception as e: diff --git a/channels/mattermost.py b/channels/mattermost.py index 8e22144c..51a42f91 100644 --- a/channels/mattermost.py +++ b/channels/mattermost.py @@ -28,7 +28,6 @@ MM_URL = "https://chat.singularitynet.io" CHANNEL_ID = "8fjrmabjx7gupy7e5kjznpt5qh" #NOT AN ID JUST NAME: "omegaclaw"x BOT_TOKEN = "" -BOT_USER_ID = "" def _get_bot_user_id(): global headers @@ -109,29 +108,6 @@ def _deliver_outbound(text): response.raise_for_status() -def _send_export_reply(user_id, text): - """Deliver export-control output through the requester's Mattermost DM.""" - if not BOT_USER_ID: - raise RuntimeError("Mattermost bot identity is not initialized") - direct = requests.post( - f"{MM_URL}/api/v4/channels/direct", - headers=_headers, - json=[BOT_USER_ID, user_id], - timeout=15, - ) - direct.raise_for_status() - direct_channel_id = str(direct.json().get("id", "")).strip() - if not direct_channel_id: - raise RuntimeError("Mattermost did not return a direct-message channel") - response = requests.post( - f"{MM_URL}/api/v4/posts", - headers=_headers, - json={"channel_id": direct_channel_id, "message": str(text)}, - timeout=15, - ) - response.raise_for_status() - - def _flush_outbox(): try: _outbox.flush(_deliver_outbound, _ready_to_send) @@ -181,10 +157,7 @@ def _ws_session(): state = _is_allowed_message(user_id, message) if state == "allow": name = _get_display_name(user_id) - owner_key = f"mattermost:{CHANNEL_ID}:{user_id}" - deliver_reply = lambda text, target=user_id: _send_export_reply(target, text) - if not channels.handle_control_message(message, owner_key, deliver_reply): - _set_last(f"{name}: {message}") + _set_last(f"{name}: {message}") elif state == "auth_bound": name = _get_display_name(user_id) send_message(f"Authentication successful for {name}.") diff --git a/channels/memory_export.py b/channels/memory_export.py index 5eb8eda9..7b8641a2 100644 --- a/channels/memory_export.py +++ b/channels/memory_export.py @@ -18,9 +18,6 @@ def _get_transfer(): global _transfer if _transfer is None: - # Keep the runtime package optional while channel modules are imported. - # The Docker image installs it, but lightweight CI/unit environments do - # not need it unless an export is actually executed. from memory_portability import MemoryTransfer embedding_provider = str(config_get_by_key("embeddingprovider", "Local")).strip() diff --git a/channels/slack.py b/channels/slack.py index 761c9b24..daa05dfc 100644 --- a/channels/slack.py +++ b/channels/slack.py @@ -97,7 +97,7 @@ def _download_file(url, timeout=30): logger.exception(f"Slack file download failed: {url}") attach = None return attach - + def _set_last(msg): global _last_message with _msg_lock: @@ -408,44 +408,41 @@ def _poll_channel(channel_id): state = _is_allowed_message(channel_id, user_id, text) display_name = _get_display_name(user_id) if state == "allow": - owner_key = f"slack:{channel_id}:{user_id}" - deliver_reply = lambda message, target=user_id: _send_export_reply(target, message) - if not channels.handle_control_message(text, owner_key, deliver_reply): - # After the user is validated, expose attachments to the agent. - if files: - file_info = [] - for f in files: - name = f.get("name", "unknown") - url = f.get("url_private_download") or f.get("url_private", "") - mime = f.get("mimetype", "") - size = f.get("size", 0) - file_info.append(f"[ATTACHMENT: {name} | {mime} | {size} bytes | {url}]") - # Download content to /tmp for agent access, check file size first -- must be below maximum. - if url: - if size <= SL_MAX_FILE_SIZE_BYTES: - file_data = _download_file(url, timeout=30) - if file_data: - safe_name = name.replace("/", "_") - tmp_path = f"/tmp/slack_attachment_{safe_name}" - try: - with open(tmp_path, "wb") as fh: - fh.write(file_data) - file_info.append(f"[SAVED: {tmp_path}]") - except Exception as exc: - logger.exception(f"Failed to save Slack attachment: {exc}") - file_info.append(f"[ATTACHMENT DOWNLOAD FAILED: {name} {exc}]") - else: - file_info.append(f"[ATTACHMENT DOWNLOAD FAILED, NO DATA: {name}]") + # after user validated, read attachments + if files: + file_info = [] + for f in files: + name = f.get("name", "unknown") + url = f.get("url_private_download") or f.get("url_private", "") + mime = f.get("mimetype", "") + size = f.get("size", 0) + file_info.append(f"[ATTACHMENT: {name} | {mime} | {size} bytes | {url}]") + # Download content to /tmp for agent access, check file size first -- must be below maximum. + if url: + if size <= SL_MAX_FILE_SIZE_BYTES: + file_data = _download_file(url, timeout=30) + if file_data: + safe_name = name.replace("/", "_") + tmp_path = f"/tmp/slack_attachment_{safe_name}" + try: + with open(tmp_path, "wb") as fh: + fh.write(file_data) + file_info.append(f"[SAVED: {tmp_path}]") + except Exception as exc: + logger.exception(f"Failed to save Slack attachment: {exc}") + file_info.append(f"[ATTACHMENT DOWNLOAD FAILED: {name} {exc}]") else: - file_info.append(f"[ATTACHMENT DOWNLOAD SIZE TOO LARGE, FAILED: {name} Size: {size}]") + file_info.append(f"[ATTACHMENT DOWNLOAD FAILED, NO DATA: {name}]") else: - file_info.append(f"[ATTACHMENT DOWNLOAD BAD URL, FAILED: {name} url: {url}]") - if text: - text = text + "\n" + "\n".join(file_info) + file_info.append(f"[ATTACHMENT DOWNLOAD SIZE TOO LARGE, FAILED: {name} Size: {size}]") else: - text = "\n".join(file_info) + file_info.append(f"[ATTACHMENT DOWNLOAD BAD URL, FAILED: {name} url: {url}]") + if text: + text = text + "\n" + "\n".join(file_info) + else: + text = "\n".join(file_info) - _set_last(f"<@{user_id}> ({display_name}): {text}") + _set_last(f"<@{user_id}> ({display_name}): {text}") elif state == "auth_bound": send_message(f"Authentication successful for {display_name}.") @@ -471,15 +468,6 @@ def _deliver_outbound(chunk): ) -def _send_export_reply(user_id, text): - """Deliver export-control output through the requester's Slack DM.""" - payload = _api_call("conversations.open", {"users": user_id}, timeout=15) - channel_id = str((payload.get("channel") or {}).get("id", "")).strip() - if not channel_id: - raise RuntimeError("Slack did not return a direct-message channel") - _api_call("chat.postMessage", {"channel": channel_id, "text": str(text)}, timeout=15) - - def _flush_outbox(): try: _outbox.flush(_deliver_outbound, _ready_to_send) diff --git a/channels/telegram.py b/channels/telegram.py index c8cbbfd9..331da7ff 100644 --- a/channels/telegram.py +++ b/channels/telegram.py @@ -163,16 +163,6 @@ def _deliver_outbound(chunk): ) -def _send_export_reply(user_id, text): - """Send private export-control output to the authenticated requester.""" - _api_call( - "sendMessage", - {"chat_id": user_id, "text": str(text)}, - timeout=15, - use_post=True, - ) - - def _flush_outbox(): global _connected try: @@ -221,10 +211,7 @@ def _poll_loop(): state = _is_allowed_message(chat_id, user_id, text) display_name = _display_name(user, chat) if state == "allow": - owner_key = f"telegram:{chat_id}:{user_id}" - deliver_reply = lambda message, target=user_id: _send_export_reply(target, message) - if not channels.handle_control_message(text, owner_key, deliver_reply): - _set_last(f"{display_name}: {text}") + _set_last(f"{display_name}: {text}") elif state == "auth_bound": send_message(f"Authentication successful for {display_name}.") _flush_outbox() diff --git a/channels/wschat.py b/channels/wschat.py index e90c848f..d188a58a 100644 --- a/channels/wschat.py +++ b/channels/wschat.py @@ -70,7 +70,6 @@ sys.path.insert(0, str(_REPO_ROOT)) from src.logger import get_logger -from memory_export import is_export_command try: import channels except ModuleNotFoundError: @@ -195,9 +194,6 @@ def _handle_frame(raw_message): if not isinstance(seq, int) or not isinstance(text, str): logger.warning(f"Ignoring malformed user_message frame: {frame!r}") return - if is_export_command(text): - send_message("Memory export is not supported on the WebSocket channel.") - return _enqueue_user_message(seq, text) return diff --git a/src/channels.py b/src/channels.py index 90e2b52d..04c5d215 100644 --- a/src/channels.py +++ b/src/channels.py @@ -4,16 +4,36 @@ _commChannelRegistry = {} -def handle_control_message(text: str, owner_key: str, deliver_reply) -> bool: + +def handle_control_message(message: str) -> bool: + from auth import get_channel_authenticated_user_id, is_auth_enabled from memory_export import handle_export_command, is_export_command - if not is_export_command(text): + sender, separator, command = message.rpartition(": ") + if not separator: + command = message + if not is_export_command(command): return False - reply = handle_export_command(text, owner_key) + + authenticated_user_id = None + if is_auth_enabled(): + authenticated_user_id = get_channel_authenticated_user_id( + _commchannel_id.upper() + ) + owner = authenticated_user_id or sender.strip() or "owner" + owner_key = f"{_commchannel_id}:{owner}" + if _commchannel_id == "websocket": + reply = "Memory export is not supported on the WebSocket channel." + else: + reply = handle_export_command(command, owner_key) if reply is not None: - deliver_reply(reply) + try: + _commchannel.send(reply) + except Exception as exc: + logger.exception("Failed to deliver control-message response: %s", exc) return True + class CommChannel: """Communication channel implementation""" @@ -40,20 +60,25 @@ def registerCommChannel(id: str, channel: CommChannel) -> None: _commChannelRegistry[id] = channel _commchannel: CommChannel = None +_commchannel_id = "" def commChannelStart(commchannel): """Select and start one of the communication channels registered by plugins""" - global _commchannel + global _commchannel, _commchannel_id _commchannel = _commChannelRegistry.get(commchannel, None) if _commchannel is None: _error("commChannelStart", f"Communication channel plugin {commchannel} is not registered") + _commchannel_id = str(commchannel).lower() _commchannel.start() def commChannelReceive(): """Receive message from selected communication channel""" global _commchannel - return _commchannel.receive() + messages = _commchannel.receive().split(" | ") + return " | ".join( + message for message in messages if not handle_control_message(message) + ) def commChannelSend(message): """Send message via selected communication channel""" diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index f9e64842..6f891b25 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -94,19 +94,16 @@ def __init__(self, transfer_dir): assert isinstance(transfer, FakeTransfer) assert created == [(handler._TRANSFER_DIR, "OpenAI")] -def test_websocket_ignores_memory_export(monkeypatch): +def test_websocket_defers_memory_export_to_core_dispatch(monkeypatch): config = types.ModuleType("config") config.config_get_by_key = lambda key, default=None: default logger_mod = types.ModuleType("src.logger") logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) - control = types.ModuleType("memory_export") - control.is_export_command = lambda text: text.startswith("/memory-export") channels = types.ModuleType("channels") channels.CommChannel = object channels.registerCommChannel = lambda *args: None monkeypatch.setitem(sys.modules, "config", config) monkeypatch.setitem(sys.modules, "src.logger", logger_mod) - monkeypatch.setitem(sys.modules, "memory_export", control) monkeypatch.setitem(sys.modules, "channels", channels) spec = importlib.util.spec_from_file_location( @@ -122,21 +119,123 @@ def test_websocket_ignores_memory_export(monkeypatch): module._handle_frame(json.dumps({ "type": "user_message", "seq": 1, "text": "/memory-export both" })) - assert received == [] + assert received == [(1, "/memory-export both")] + assert replies == [] + + +def test_commchannel_receive_dispatches_control_commands(monkeypatch): + authenticated_user_id = "telegram-user-123" + auth = types.ModuleType("auth") + auth.is_auth_enabled = lambda: True + auth.get_channel_authenticated_user_id = lambda channel: ( + authenticated_user_id if channel == "TELEGRAM" else None + ) + monkeypatch.setitem(sys.modules, "auth", auth) + + owners: list[str] = [] + control = types.ModuleType("memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda text, owner: ( + owners.append(owner) or "Export requested" + ) + monkeypatch.setitem(sys.modules, "memory_export", control) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + + replies: list[str] = [] + channels._commchannel = types.SimpleNamespace( + receive=lambda: "alice: /memory-export both | alice: hello", + send=replies.append, + ) + channels._commchannel_id = "telegram" + + assert channels.commChannelReceive() == "alice: hello" + assert replies == ["Export requested"] + assert owners == [f"telegram:{authenticated_user_id}"] + + +def test_commchannel_receive_rejects_unsupported_control_channel(monkeypatch): + auth = types.ModuleType("auth") + auth.is_auth_enabled = lambda: True + auth.get_channel_authenticated_user_id = lambda *_: "websocket-user" + monkeypatch.setitem(sys.modules, "auth", auth) + + control = types.ModuleType("memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda *_: pytest.fail( + "unsupported channels must not execute exports" + ) + monkeypatch.setitem(sys.modules, "memory_export", control) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + + replies: list[str] = [] + channels._commchannel = types.SimpleNamespace( + receive=lambda: "/memory-export both", + send=replies.append, + ) + channels._commchannel_id = "websocket" + + assert channels.commChannelReceive() == "" assert replies == ["Memory export is not supported on the WebSocket channel."] -def test_shared_dispatcher_consumes_control_commands(monkeypatch): + +def test_commchannel_receive_does_not_consume_command_mentions(monkeypatch): + auth = types.ModuleType("auth") + auth.is_auth_enabled = lambda: False + auth.get_channel_authenticated_user_id = lambda *_: pytest.fail( + "disabled authentication must not read a persisted owner" + ) + monkeypatch.setitem(sys.modules, "auth", auth) + control = types.ModuleType("memory_export") control.is_export_command = lambda text: text == "/memory-export both" - control.handle_export_command = lambda text, owner: "Export requested" + control.handle_export_command = lambda *_: pytest.fail( + "a command mentioned in normal text must not execute" + ) monkeypatch.setitem(sys.modules, "memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) channels = importlib.import_module("channels") - replies = [] - assert channels.handle_control_message( - "/memory-export both", "telegram:chat:user", replies.append + message = "alice: please use /memory-export both" + channels._commchannel = types.SimpleNamespace( + receive=lambda: message, + send=lambda *_: pytest.fail("normal messages must not generate replies"), + ) + channels._commchannel_id = "telegram" + + assert channels.commChannelReceive() == message + + +def test_commchannel_receive_falls_back_to_sender_without_auth(monkeypatch): + auth = types.ModuleType("auth") + auth.is_auth_enabled = lambda: False + auth.get_channel_authenticated_user_id = lambda *_: pytest.fail( + "disabled authentication must not read a persisted owner" + ) + monkeypatch.setitem(sys.modules, "auth", auth) + + owners: list[str] = [] + control = types.ModuleType("memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda text, owner: ( + owners.append(owner) or "Export requested" ) + monkeypatch.setitem(sys.modules, "memory_export", control) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + + replies: list[str] = [] + channels._commchannel = types.SimpleNamespace( + receive=lambda: "alice: /memory-export both", + send=replies.append, + ) + channels._commchannel_id = "telegram" + + assert channels.commChannelReceive() == "" assert replies == ["Export requested"] - assert not channels.handle_control_message("hello", "telegram:chat:user", replies.append) + assert owners == ["telegram:alice"] From 89021a8dd39cd93349569e7fa37347048c4d3870 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Sun, 23 Aug 2026 00:27:11 +0300 Subject: [PATCH 25/34] chore: moved memory_export adapter into src dir --- src/channels.py | 2 +- {channels => src}/memory_export.py | 10 ++++++++++ tests/test_memory_export.py | 18 +++++++++--------- 3 files changed, 20 insertions(+), 10 deletions(-) rename {channels => src}/memory_export.py (99%) diff --git a/src/channels.py b/src/channels.py index 04c5d215..a629f6b2 100644 --- a/src/channels.py +++ b/src/channels.py @@ -7,7 +7,7 @@ def handle_control_message(message: str) -> bool: from auth import get_channel_authenticated_user_id, is_auth_enabled - from memory_export import handle_export_command, is_export_command + from src.memory_export import handle_export_command, is_export_command sender, separator, command = message.rpartition(": ") if not separator: diff --git a/channels/memory_export.py b/src/memory_export.py similarity index 99% rename from channels/memory_export.py rename to src/memory_export.py index 7b8641a2..7d0e5202 100644 --- a/channels/memory_export.py +++ b/src/memory_export.py @@ -15,6 +15,7 @@ _transfer = None + def _get_transfer(): global _transfer if _transfer is None: @@ -28,6 +29,7 @@ def _get_transfer(): _transfer = MemoryTransfer(_TRANSFER_DIR) return _transfer + _TOKEN_TTL_SECONDS = 60 _request_lock = threading.Lock() @@ -35,12 +37,14 @@ def _get_transfer(): _VALID_COMPONENTS = ("history", "ltm", "both") + def is_export_enabled() -> bool: value = config_get_by_key("memoryExportEnabled", False) return value is True or ( isinstance(value, str) and value.strip().lower() == "true" ) + def is_export_command(text: str) -> bool: command = text.strip().split(None, 1) if not command: @@ -48,10 +52,12 @@ def is_export_command(text: str) -> bool: name = command[0].lower() return name == "/memory-export" or name.startswith("/memory-export@") + def _command_arguments(text: str) -> str: parts = text.strip().split(None, 1) return parts[1].strip() if len(parts) == 2 else "" + def _issue_token(owner_key: str, component: str) -> str: token = secrets.token_hex(8) _pending_requests[owner_key] = ( @@ -61,6 +67,7 @@ def _issue_token(owner_key: str, component: str) -> str: ) return token + def handle_export_command( text: str, owner_key: str = "default-owner", @@ -90,6 +97,7 @@ def handle_export_command( "/memory-export confirm " ) + def _handle_request(owner_key: str, component: str) -> str: with _request_lock: token = _issue_token(owner_key, component) @@ -100,6 +108,7 @@ def _handle_request(owner_key: str, component: str) -> str: f"/memory-export confirm {token}" ) + def _handle_confirm(owner_key: str, token: str) -> str: if not token: return "Usage: /memory-export confirm " @@ -122,6 +131,7 @@ def _handle_confirm(owner_key: str, token: str) -> str: logger.exception(f"memory_export: export failed: {exc}") return f"Memory export failed: {exc}" + def _format_export(result: dict) -> str: return ( "Memory export complete\n" diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index 6f891b25..0ae2aa0e 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -25,7 +25,7 @@ def handler(monkeypatch): spec = importlib.util.spec_from_file_location( "memory_export_under_test", - REPO_ROOT / "channels" / "memory_export.py", + REPO_ROOT / "src" / "memory_export.py", ) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) @@ -133,12 +133,12 @@ def test_commchannel_receive_dispatches_control_commands(monkeypatch): monkeypatch.setitem(sys.modules, "auth", auth) owners: list[str] = [] - control = types.ModuleType("memory_export") + control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" control.handle_export_command = lambda text, owner: ( owners.append(owner) or "Export requested" ) - monkeypatch.setitem(sys.modules, "memory_export", control) + monkeypatch.setitem(sys.modules, "src.memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) channels = importlib.import_module("channels") @@ -161,12 +161,12 @@ def test_commchannel_receive_rejects_unsupported_control_channel(monkeypatch): auth.get_channel_authenticated_user_id = lambda *_: "websocket-user" monkeypatch.setitem(sys.modules, "auth", auth) - control = types.ModuleType("memory_export") + control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" control.handle_export_command = lambda *_: pytest.fail( "unsupported channels must not execute exports" ) - monkeypatch.setitem(sys.modules, "memory_export", control) + monkeypatch.setitem(sys.modules, "src.memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) channels = importlib.import_module("channels") @@ -190,12 +190,12 @@ def test_commchannel_receive_does_not_consume_command_mentions(monkeypatch): ) monkeypatch.setitem(sys.modules, "auth", auth) - control = types.ModuleType("memory_export") + control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" control.handle_export_command = lambda *_: pytest.fail( "a command mentioned in normal text must not execute" ) - monkeypatch.setitem(sys.modules, "memory_export", control) + monkeypatch.setitem(sys.modules, "src.memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) channels = importlib.import_module("channels") @@ -219,12 +219,12 @@ def test_commchannel_receive_falls_back_to_sender_without_auth(monkeypatch): monkeypatch.setitem(sys.modules, "auth", auth) owners: list[str] = [] - control = types.ModuleType("memory_export") + control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" control.handle_export_command = lambda text, owner: ( owners.append(owner) or "Export requested" ) - monkeypatch.setitem(sys.modules, "memory_export", control) + monkeypatch.setitem(sys.modules, "src.memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) channels = importlib.import_module("channels") From d5c623974debcff11fcc3308e7345c0ad6e01ecc Mon Sep 17 00:00:00 2001 From: CodersKin Date: Mon, 24 Aug 2026 14:51:02 +0300 Subject: [PATCH 26/34] chore: updated version for documentation --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ff1cc221..8ace71d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ chromadb==1.5.9 openai==2.38.0 transformers==5.8.0 sentence-transformers==5.5.1 -import-kb==0.1.9 +import-kb==0.2.0 py-landlock==0.1.1 pyyaml==6.0.3 ddgs==9.14.4 From f2a26c6a742f9a9c06585de26cb25d9de5773724 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Tue, 25 Aug 2026 10:14:35 +0300 Subject: [PATCH 27/34] Feat: enabled support for wsocket channel export and removed redundant confirmation --- config/config.yaml | 2 +- docs/reference-channels.md | 1 + docs/reference-memory-portability.md | 19 +-- src/channels.py | 39 ++++-- src/memory_export.py | 64 ++-------- tests/test_memory_export.py | 178 +++++++++++++++++---------- 6 files changed, 157 insertions(+), 146 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 37f0b988..cce42d77 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -36,7 +36,7 @@ maxEpisodeRecallLines: 20 maxHistory: 30000 # `Local` (Python-side model) or `OpenAI` (requires `OPENAI_API_KEY`) embeddingprovider: Local -# Enable authenticated /memory-export commands (disabled by default). +# Enable authenticated operator-triggered /memory-export commands (disabled by default). memoryExportEnabled: false # Policy diff --git a/docs/reference-channels.md b/docs/reference-channels.md index 7d8b69d8..539b229d 100644 --- a/docs/reference-channels.md +++ b/docs/reference-channels.md @@ -74,6 +74,7 @@ Minimal JSON chat adapter over a WebSocket connection. Selected with `commchanne - `stop_websocket()` — stop the listener thread and close the socket. - Requires the `websockets` Python package. - When `WS_TOKEN` is set it is sent as an `Authorization: Bearer ` header. Unlike the IRC/Telegram/Slack adapters there is no one-time `auth ` gate — trust is established by the endpoint URL and bearer token. +- Supports immediate `/memory-export history|ltm|both` commands when memory export is enabled and `WS_TOKEN` is configured. The export handler uses a SHA-256-derived connection principal and never exposes the bearer token. Protect the endpoint with `wss://` and server-side access controls because WebSocket does not have the per-user ownership gate used by the other channels. - Reconnects automatically with exponential backoff (1s → 30s, ±20% jitter) and is safe to start once at process startup. ### Frame protocol diff --git a/docs/reference-memory-portability.md b/docs/reference-memory-portability.md index 4d44f6a1..a4758feb 100644 --- a/docs/reference-memory-portability.md +++ b/docs/reference-memory-portability.md @@ -21,20 +21,20 @@ The transfer directory must be writable by the container's agent user. ## Export -In a supported chat, request one component and confirm the returned short-lived -token. When channel authentication is active, only the authenticated owner can -use these commands. When it is disabled, normal channel access rules apply: +In the active chat, request one component. The export runs immediately. For IRC, +Telegram, Slack, and Mattermost, the export handler requires the authenticated +user ID persisted by the channel authorization layer. WebSocket export requires +a configured `WS_TOKEN`; the handler derives a non-reversible principal from the +token so the credential itself is never used as an identifier: ```text /memory-export history /memory-export ltm /memory-export both -/memory-export confirm ``` -The confirmed export runs immediately in the channel harness. Completion is -delivered to the requester and includes the filename, record count, size, and -SHA-256. +Completion is delivered through the active channel and includes the filename, +record count, size, and SHA-256. Archives contain selected persistent user memory only: @@ -71,7 +71,8 @@ counts, and embedding compatibility before changing live memory. It runs before the agent loop starts. A receipt prevents a completed archive import from running again on container restart. -## Limits +## Security -Memory export commands are not supported on the WebSocket chat channel. Archives are private operator data; keep the host transfer directory protected. +Memory export is denied when channel authentication is disabled, no authenticated +channel user has been persisted, or WebSocket has no `WS_TOKEN`. diff --git a/src/channels.py b/src/channels.py index a629f6b2..504a7217 100644 --- a/src/channels.py +++ b/src/channels.py @@ -1,3 +1,4 @@ +import hashlib import logging logger = logging.getLogger(__name__) @@ -5,27 +6,39 @@ _commChannelRegistry = {} -def handle_control_message(message: str) -> bool: +def _authenticated_export_principal() -> str | None: + if _commchannel_id == "websocket": + from config import config_get_by_key + + token = str(config_get_by_key("WS_TOKEN", "")).strip() + if not token: + return None + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return f"websocket:{digest}" + from auth import get_channel_authenticated_user_id, is_auth_enabled + + if not is_auth_enabled(): + return None + return get_channel_authenticated_user_id(_commchannel_id.upper()) + + +def handle_control_message(message: str) -> bool: from src.memory_export import handle_export_command, is_export_command - sender, separator, command = message.rpartition(": ") + _, separator, command = message.rpartition(": ") if not separator: command = message if not is_export_command(command): return False - authenticated_user_id = None - if is_auth_enabled(): - authenticated_user_id = get_channel_authenticated_user_id( - _commchannel_id.upper() - ) - owner = authenticated_user_id or sender.strip() or "owner" - owner_key = f"{_commchannel_id}:{owner}" - if _commchannel_id == "websocket": - reply = "Memory export is not supported on the WebSocket channel." - else: - reply = handle_export_command(command, owner_key) + try: + authenticated_principal = _authenticated_export_principal() + except Exception as exc: + logger.exception("Failed to resolve memory-export principal: %s", exc) + authenticated_principal = None + + reply = handle_export_command(command, authenticated_principal) if reply is not None: try: _commchannel.send(reply) diff --git a/src/memory_export.py b/src/memory_export.py index 7d0e5202..3618f358 100644 --- a/src/memory_export.py +++ b/src/memory_export.py @@ -1,9 +1,6 @@ """Shared /memory-export command handling.""" import os -import secrets -import threading -import time from pathlib import Path from config import config_get_by_key @@ -30,11 +27,6 @@ def _get_transfer(): return _transfer -_TOKEN_TTL_SECONDS = 60 - -_request_lock = threading.Lock() -_pending_requests: dict[str, tuple[str, str, float]] = {} - _VALID_COMPONENTS = ("history", "ltm", "both") @@ -58,19 +50,9 @@ def _command_arguments(text: str) -> str: return parts[1].strip() if len(parts) == 2 else "" -def _issue_token(owner_key: str, component: str) -> str: - token = secrets.token_hex(8) - _pending_requests[owner_key] = ( - token, - component, - time.monotonic() + _TOKEN_TTL_SECONDS, - ) - return token - - def handle_export_command( text: str, - owner_key: str = "default-owner", + authenticated_user_id: str | None = None, ) -> str | None: stripped = text.strip() @@ -80,50 +62,22 @@ def handle_export_command( if not is_export_enabled(): return None - rest = _command_arguments(stripped) - parts = rest.split(None, 1) - sub = parts[0].lower() if parts else "" - arg = parts[1].strip() if len(parts) > 1 else "" + if not authenticated_user_id: + return "Memory export denied: an authenticated user is required." - if sub in _VALID_COMPONENTS: - return _handle_request(owner_key, sub) + rest = _command_arguments(stripped) + component = rest.lower() - if sub == "confirm": - return _handle_confirm(owner_key, arg) + if component in _VALID_COMPONENTS: + return _export(component) return ( "Unknown /memory-export command. " - "Use: /memory-export history|ltm|both or " - "/memory-export confirm " - ) - - -def _handle_request(owner_key: str, component: str) -> str: - with _request_lock: - token = _issue_token(owner_key, component) - logger.info(f"memory_export: issued confirmation token for component={component}") - return ( - f"Export requested for: {component}\n" - f"Confirm within {_TOKEN_TTL_SECONDS}s:\n" - f"/memory-export confirm {token}" + "Use: /memory-export history|ltm|both" ) -def _handle_confirm(owner_key: str, token: str) -> str: - if not token: - return "Usage: /memory-export confirm " - - with _request_lock: - pending = _pending_requests.get(owner_key) - if pending is None: - return "No pending export request. Start with /memory-export history|ltm|both" - expected_token, component, expires_at = pending - if time.monotonic() > expires_at: - del _pending_requests[owner_key] - return "Confirmation token expired. Start again with /memory-export history|ltm|both" - if not secrets.compare_digest(expected_token, token): - return "Invalid token." - del _pending_requests[owner_key] +def _export(component: str) -> str: try: result = _get_transfer().export(component) return _format_export(result) diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index 0ae2aa0e..979dddbe 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -1,9 +1,9 @@ import importlib import importlib.util +import hashlib import json import os import sys -import time import types from pathlib import Path @@ -13,10 +13,6 @@ @pytest.fixture def handler(monkeypatch): - auth = types.ModuleType("auth") - auth.is_auth_enabled = lambda: True - monkeypatch.setitem(sys.modules, "auth", auth) - logger_mod = types.ModuleType("src.logger") logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) monkeypatch.setitem(sys.modules, "src.logger", logger_mod) @@ -32,30 +28,44 @@ def handler(monkeypatch): module.is_export_enabled = lambda: True return module -def test_export_command_requires_policy_but_not_auth(handler): - assert "Export requested" in handler.handle_export_command("/memory-export both") +def test_export_command_requires_policy(handler): + exported = [] + handler._get_transfer = lambda: types.SimpleNamespace( + export=lambda component: exported.append(component) or { + "filename": "memory.tar.gz", + "size": 1, + "sha256": "abc", + "record_count": 1, + } + ) + + assert "Memory export complete" in handler.handle_export_command( + "/memory-export both", "authenticated-user" + ) + assert exported == ["both"] + handler.is_export_enabled = lambda: False - assert handler.handle_export_command("/memory-export both") is None + assert handler.handle_export_command( + "/memory-export both", "authenticated-user" + ) is None + assert exported == ["both"] -def test_module_import_does_not_require_memory_portability(handler): - assert "memory_portability" not in sys.modules - assert handler.is_export_command("/memory-export both") -def test_expired_and_other_owner_tokens_cannot_start_export(handler): - token = handler.handle_export_command("/memory-export history", "owner-a").split()[-1] - assert "Invalid token" in handler.handle_export_command( - "/memory-export confirm wrong", "owner-a" +def test_export_requires_authenticated_user(handler): + handler._get_transfer = lambda: pytest.fail( + "an unauthenticated command must not start an export" ) - assert "No pending export" in handler.handle_export_command( - f"/memory-export confirm {token}", "owner-b" + + assert handler.handle_export_command("/memory-export both") == ( + "Memory export denied: an authenticated user is required." ) - token_state = handler._pending_requests["owner-a"] - handler._pending_requests["owner-a"] = (*token_state[:2], time.monotonic() - 1) - assert "expired" in handler.handle_export_command( - f"/memory-export confirm {token}", "owner-a" - ).lower() -def test_confirmation_exports_immediately(handler): +def test_module_import_does_not_require_memory_portability(handler): + assert "memory_portability" not in sys.modules + assert handler.is_export_command("/memory-export both") + +@pytest.mark.parametrize("component", ["history", "ltm", "both"]) +def test_export_runs_immediately(handler, component): exported = [] handler._get_transfer = lambda: types.SimpleNamespace( export=lambda component: exported.append(component) or { @@ -65,12 +75,28 @@ def test_confirmation_exports_immediately(handler): "record_count": 1, } ) - token = handler.handle_export_command("/memory-export both", "owner-a").split()[-1] - reply = handler.handle_export_command(f"/memory-export confirm {token}", "owner-a") - assert exported == ["both"] + reply = handler.handle_export_command( + f"/memory-export {component}", "authenticated-user" + ) + assert exported == [component] assert "memory.tar.gz" in reply assert "SHA-256: abc" in reply + +def test_confirmation_command_is_no_longer_supported(handler): + handler._get_transfer = lambda: pytest.fail( + "the removed confirmation command must not start an export" + ) + + reply = handler.handle_export_command( + "/memory-export confirm old-token", "authenticated-user" + ) + + assert reply == ( + "Unknown /memory-export command. " + "Use: /memory-export history|ltm|both" + ) + def test_transfer_uses_effective_runtime_embedding_provider(handler, monkeypatch): created = [] @@ -132,11 +158,11 @@ def test_commchannel_receive_dispatches_control_commands(monkeypatch): ) monkeypatch.setitem(sys.modules, "auth", auth) - owners: list[str] = [] + principals: list[str] = [] control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" - control.handle_export_command = lambda text, owner: ( - owners.append(owner) or "Export requested" + control.handle_export_command = lambda text, principal: ( + principals.append(principal) or "Memory export complete" ) monkeypatch.setitem(sys.modules, "src.memory_export", control) @@ -151,20 +177,24 @@ def test_commchannel_receive_dispatches_control_commands(monkeypatch): channels._commchannel_id = "telegram" assert channels.commChannelReceive() == "alice: hello" - assert replies == ["Export requested"] - assert owners == [f"telegram:{authenticated_user_id}"] + assert principals == [authenticated_user_id] + assert replies == ["Memory export complete"] -def test_commchannel_receive_rejects_unsupported_control_channel(monkeypatch): +def test_commchannel_receive_denies_export_without_authenticated_user(monkeypatch): auth = types.ModuleType("auth") - auth.is_auth_enabled = lambda: True - auth.get_channel_authenticated_user_id = lambda *_: "websocket-user" + auth.is_auth_enabled = lambda: False + auth.get_channel_authenticated_user_id = lambda *_: pytest.fail( + "disabled authentication must not resolve a user ID" + ) monkeypatch.setitem(sys.modules, "auth", auth) control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" - control.handle_export_command = lambda *_: pytest.fail( - "unsupported channels must not execute exports" + control.handle_export_command = lambda text, principal: ( + "Memory export denied: an authenticated user is required." + if principal is None + else pytest.fail("an unauthenticated command received a principal") ) monkeypatch.setitem(sys.modules, "src.memory_export", control) @@ -173,69 +203,81 @@ def test_commchannel_receive_rejects_unsupported_control_channel(monkeypatch): replies: list[str] = [] channels._commchannel = types.SimpleNamespace( - receive=lambda: "/memory-export both", + receive=lambda: "alice: /memory-export both", send=replies.append, ) - channels._commchannel_id = "websocket" + channels._commchannel_id = "telegram" assert channels.commChannelReceive() == "" - assert replies == ["Memory export is not supported on the WebSocket channel."] + assert replies == ["Memory export denied: an authenticated user is required."] -def test_commchannel_receive_does_not_consume_command_mentions(monkeypatch): - auth = types.ModuleType("auth") - auth.is_auth_enabled = lambda: False - auth.get_channel_authenticated_user_id = lambda *_: pytest.fail( - "disabled authentication must not read a persisted owner" +def test_commchannel_receive_dispatches_websocket_export(monkeypatch): + websocket_token = "private-websocket-token" + config = types.ModuleType("config") + config.config_get_by_key = lambda key, default=None: ( + websocket_token if key == "WS_TOKEN" else default ) - monkeypatch.setitem(sys.modules, "auth", auth) + monkeypatch.setitem(sys.modules, "config", config) + commands: list[str] = [] + principals: list[str] = [] control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" - control.handle_export_command = lambda *_: pytest.fail( - "a command mentioned in normal text must not execute" + control.handle_export_command = lambda text, principal: ( + commands.append(text) + or principals.append(principal) + or "Memory export complete" ) monkeypatch.setitem(sys.modules, "src.memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) channels = importlib.import_module("channels") - message = "alice: please use /memory-export both" + replies: list[str] = [] channels._commchannel = types.SimpleNamespace( - receive=lambda: message, - send=lambda *_: pytest.fail("normal messages must not generate replies"), + receive=lambda: "/memory-export both", + send=replies.append, ) - channels._commchannel_id = "telegram" + channels._commchannel_id = "websocket" - assert channels.commChannelReceive() == message + assert channels.commChannelReceive() == "" + assert commands == ["/memory-export both"] + assert principals == [ + f"websocket:{hashlib.sha256(websocket_token.encode('utf-8')).hexdigest()}" + ] + assert websocket_token not in principals[0] + assert replies == ["Memory export complete"] -def test_commchannel_receive_falls_back_to_sender_without_auth(monkeypatch): - auth = types.ModuleType("auth") - auth.is_auth_enabled = lambda: False - auth.get_channel_authenticated_user_id = lambda *_: pytest.fail( - "disabled authentication must not read a persisted owner" - ) - monkeypatch.setitem(sys.modules, "auth", auth) +def test_websocket_export_requires_bearer_token(monkeypatch): + config = types.ModuleType("config") + config.config_get_by_key = lambda key, default=None: default + monkeypatch.setitem(sys.modules, "config", config) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + channels._commchannel_id = "websocket" + + assert channels._authenticated_export_principal() is None + - owners: list[str] = [] +def test_commchannel_receive_does_not_consume_command_mentions(monkeypatch): control = types.ModuleType("src.memory_export") control.is_export_command = lambda text: text == "/memory-export both" - control.handle_export_command = lambda text, owner: ( - owners.append(owner) or "Export requested" + control.handle_export_command = lambda *_: pytest.fail( + "a command mentioned in normal text must not execute" ) monkeypatch.setitem(sys.modules, "src.memory_export", control) monkeypatch.delitem(sys.modules, "channels", raising=False) channels = importlib.import_module("channels") - replies: list[str] = [] + message = "alice: please use /memory-export both" channels._commchannel = types.SimpleNamespace( - receive=lambda: "alice: /memory-export both", - send=replies.append, + receive=lambda: message, + send=lambda *_: pytest.fail("normal messages must not generate replies"), ) channels._commchannel_id = "telegram" - assert channels.commChannelReceive() == "" - assert replies == ["Export requested"] - assert owners == ["telegram:alice"] + assert channels.commChannelReceive() == message From a3f09438d942913ae5d4d445b2a7aec1afcad169 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Tue, 25 Aug 2026 10:17:08 +0300 Subject: [PATCH 28/34] chore: updated the readme --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index eda541e7..69c918cd 100644 --- a/README.md +++ b/README.md @@ -102,11 +102,10 @@ docker volume rm omegaclaw-memory Memory export is disabled by default. See the [memory portability reference](./docs/reference-memory-portability.md) for setup, export controls, archive contents, and import modes. -> **Current limitation:** Memory import and interrupted-import recovery are -> supported only when OmegaClaw is run through Docker using -> `scripts/omegaclaw`. Standalone execution does not invoke the container -> entrypoint and therefore does not support memory import or recovery in this -> release. +> **Current limitation:** Memory import does not work in a standalone OmegaClaw +> run. Import and interrupted-import recovery are supported only through Docker +> using `scripts/omegaclaw`, because both operations run from the container +> entrypoint before the agent loop starts. To restore an archive while upgrading to a tagged image, use the same transfer directory: From d862554bab3104a42dae21384f7ee90e89b18222 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Tue, 25 Aug 2026 11:36:16 +0300 Subject: [PATCH 29/34] chore: updated the import-kb version to the latest patch --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8ace71d5..d819dcb0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ chromadb==1.5.9 openai==2.38.0 transformers==5.8.0 sentence-transformers==5.5.1 -import-kb==0.2.0 +import-kb==0.2.1 py-landlock==0.1.1 pyyaml==6.0.3 ddgs==9.14.4 From acf9230dfcdd15a86e83a7a9c35933da9e138d96 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Mon, 31 Aug 2026 15:02:43 +0300 Subject: [PATCH 30/34] Feat: paths are now implemented as plugin config --- config/config.yaml | 6 ++- docs/reference-configuration.md | 2 + entrypoint.sh | 38 ++++++++++------ src/memory_export.py | 36 ++++++++++++++- tests/test_memory_export.py | 78 ++++++++++++++++++++++++++++++--- 5 files changed, 137 insertions(+), 23 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index b05e9bc9..4c3d0f02 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -21,8 +21,8 @@ wakeupInterval: 600 # Path to the logger configuration file # See https://docs.python.org/3/library/logging.config.html#configuration-file-format logConfigPath: "" -# (internal) Path to the memory directory -#memoryDirectory: "./" +# Directory containing persistent memory files such as history.metta. +memoryDirectory: "./repos/OmegaClaw-Core/memory" # Memory @@ -34,6 +34,8 @@ maxRecallItems: 20 maxEpisodeRecallLines: 20 # Tail of `memory/history.metta` included in the prompt (chars) maxHistory: 30000 +# ChromaDB persistence directory used for long-term memory portability. +chromaDbPath: "./chroma_db" # `Local` (Python-side model) or `OpenAI` (requires `OPENAI_API_KEY`) embeddingprovider: Local # Enable authenticated operator-triggered /memory-export commands (disabled by default). diff --git a/docs/reference-configuration.md b/docs/reference-configuration.md index ee4ffe9c..1084a58e 100644 --- a/docs/reference-configuration.md +++ b/docs/reference-configuration.md @@ -31,6 +31,8 @@ This reads a command-line override via `argk` (`name=value` on the MeTTa command | `maxRecallItems` | 20 | Items returned by `query`. | | `maxEpisodeRecallLines` | 20 | Lines returned by `episodes`. | | `maxHistory` | 30000 (chars) | Tail of `memory/history.metta` included in the prompt. | +| `memoryDirectory` | `./repos/OmegaClaw-Core/memory` | Directory containing persistent memory files such as `history.metta`. | +| `chromaDbPath` | `./chroma_db` | ChromaDB persistence directory used for memory backup and restore. | | `embeddingprovider` | `Local` | `Local` (Python-side model) or `OpenAI`. | ## Channels (`src/channels.metta`, `initChannels`) diff --git a/entrypoint.sh b/entrypoint.sh index 09e8c3b5..cf3dc404 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -40,34 +40,46 @@ if [[ "${IMPORT_KB_ON_START}" == "1" ]]; then su nobody -s /bin/sh -c "${OMEGACLAW_DIR}/scripts/import_knowledge.sh" fi -MEMORY_PORTABILITY_PYTHON='from memory_portability import MemoryTransfer; MemoryTransfer().recover()' +MEMORY_PORTABILITY_PYTHON='import os +from config import init_config +from memory_export import create_memory_store +from memory_portability import MemoryTransfer + +init_config([]) +transfer = MemoryTransfer(store=create_memory_store()) +operation = os.environ["MEMORY_PORTABILITY_OPERATION"] +if operation == "recover": + transfer.recover() +elif operation == "import": + transfer.import_archive( + os.environ["MEMORY_IMPORT_FILE"], + mode=os.environ.get("MEMORY_IMPORT_MODE", "overwrite"), + include_history=os.environ.get("MEMORY_IMPORT_NO_HISTORY") != "1", + include_vectors=os.environ.get("MEMORY_IMPORT_NO_VECTOR") != "1", + ) +else: + raise ValueError(f"Unsupported memory portability operation: {operation!r}")' export MEMORY_PORTABILITY_PYTHON +export PYTHONPATH="${OMEGACLAW_DIR}:${OMEGACLAW_DIR}/src${PYTHONPATH:+:${PYTHONPATH}}" + +export MEMORY_PORTABILITY_OPERATION=recover su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \ || { echo "Memory import recovery failed. Aborting startup." >&2; exit 1; } -unset MEMORY_PORTABILITY_PYTHON if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then echo "memory_portability: importing ${MEMORY_IMPORT_FILE}" - MEMORY_PORTABILITY_PYTHON='import os -from memory_portability import MemoryTransfer -MemoryTransfer().import_archive( - os.environ["MEMORY_IMPORT_FILE"], - mode=os.environ.get("MEMORY_IMPORT_MODE", "overwrite"), - include_history=os.environ.get("MEMORY_IMPORT_NO_HISTORY") != "1", - include_vectors=os.environ.get("MEMORY_IMPORT_NO_VECTOR") != "1", -)' - export MEMORY_PORTABILITY_PYTHON + export MEMORY_PORTABILITY_OPERATION=import su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \ || { echo "Memory import failed. Aborting startup." >&2; exit 1; } - unset MEMORY_PORTABILITY_PYTHON echo "memory_portability: import complete" fi +unset MEMORY_PORTABILITY_OPERATION MEMORY_PORTABILITY_PYTHON PYTHONPATH # Scrub environment: only allowlisted vars survive. SAFE_VARS="HOME USER PATH HOSTNAME TERM LANG LC_ALL \ PYTHONDONTWRITEBYTECODE PYTHONUNBUFFERED \ HF_HOME SENTENCE_TRANSFORMERS_HOME HF_HUB_OFFLINE TRANSFORMERS_OFFLINE \ - EMBEDDING_PROVIDER OMEGACLAW_DIR MEMORY_DIR TEST_SERVER_IP" + CHROMA_DB_PATH EMBEDDING_PROVIDER OMEGACLAW_DIR MEMORY_DIR TEST_SERVER_IP" env_args="" for var in $SAFE_VARS; do diff --git a/src/memory_export.py b/src/memory_export.py index 3618f358..7e55e7e3 100644 --- a/src/memory_export.py +++ b/src/memory_export.py @@ -4,6 +4,7 @@ from pathlib import Path from config import config_get_by_key +from helper import projectRootDirectory from src.logger import get_logger logger = get_logger(__name__) @@ -13,6 +14,36 @@ _transfer = None +def _resolve_memory_dir() -> Path: + default = os.environ.get( + "MEMORY_DIR", + str(Path(projectRootDirectory()) / "memory"), + ) + configured = config_get_by_key("memoryDirectory", default) + return Path(str(configured)).expanduser().resolve() + + +def _resolve_chroma_path() -> Path: + environment_path = os.environ.get("CHROMA_DB_PATH") + if environment_path: + return Path(environment_path).expanduser().resolve() + + default = str(Path(projectRootDirectory()).parents[1] / "chroma_db") + configured = config_get_by_key("chromaDbPath", default) + return Path(str(configured)).expanduser().resolve() + + +def create_memory_store(): + """Build an import-kb store from OmegaClaw's effective configuration.""" + from memory_portability.storage import MemoryStore + + return MemoryStore( + memory_dir=_resolve_memory_dir(), + chroma_path=_resolve_chroma_path(), + collection_name="memories", + ) + + def _get_transfer(): global _transfer if _transfer is None: @@ -23,7 +54,10 @@ def _get_transfer(): raise ValueError(f"Unsupported embedding provider: {embedding_provider!r}") os.environ["EMBEDDING_PROVIDER"] = embedding_provider - _transfer = MemoryTransfer(_TRANSFER_DIR) + _transfer = MemoryTransfer( + transfer_dir=_TRANSFER_DIR, + store=create_memory_store(), + ) return _transfer diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index 979dddbe..a85383ae 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -101,13 +101,17 @@ def test_transfer_uses_effective_runtime_embedding_provider(handler, monkeypatch created = [] class FakeTransfer: - def __init__(self, transfer_dir): - created.append((transfer_dir, os.environ["EMBEDDING_PROVIDER"])) + def __init__(self, **kwargs): + created.append({ + **kwargs, + "embedding_provider": os.environ["EMBEDDING_PROVIDER"], + }) monkeypatch.delenv("EMBEDDING_PROVIDER", raising=False) - mp_mod = types.ModuleType("memory_portability") - mp_mod.MemoryTransfer = FakeTransfer - monkeypatch.setitem(sys.modules, "memory_portability", mp_mod) + package = types.ModuleType("memory_portability") + package.MemoryTransfer = FakeTransfer + monkeypatch.setitem(sys.modules, "memory_portability", package) + monkeypatch.setattr(handler, "create_memory_store", lambda: "configured-store") monkeypatch.setattr( handler, "config_get_by_key", @@ -117,8 +121,68 @@ def __init__(self, transfer_dir): transfer = handler._get_transfer() - assert isinstance(transfer, FakeTransfer) - assert created == [(handler._TRANSFER_DIR, "OpenAI")] + assert transfer is handler._transfer + assert created == [{ + "transfer_dir": handler._TRANSFER_DIR, + "store": "configured-store", + "embedding_provider": "OpenAI", + }] + + +def test_memory_store_receives_explicit_omegaclaw_storage_configuration( + handler, + monkeypatch, + tmp_path, +): + created_stores = [] + + class FakeStore: + def __init__(self, **kwargs): + created_stores.append(kwargs) + + package = types.ModuleType("memory_portability") + package.__path__ = [] + storage = types.ModuleType("memory_portability.storage") + storage.MemoryStore = FakeStore + monkeypatch.setitem(sys.modules, "memory_portability", package) + monkeypatch.setitem(sys.modules, "memory_portability.storage", storage) + + memory_dir = tmp_path / "custom-memory" + chroma_path = tmp_path / "custom-chroma" + monkeypatch.setattr(handler, "_resolve_memory_dir", lambda: memory_dir) + monkeypatch.setattr(handler, "_resolve_chroma_path", lambda: chroma_path) + + store = handler.create_memory_store() + + assert isinstance(store, FakeStore) + assert created_stores == [ + { + "memory_dir": memory_dir, + "chroma_path": chroma_path, + "collection_name": "memories", + } + ] + + +def test_storage_paths_are_resolved_from_omegaclaw_config( + handler, + monkeypatch, + tmp_path, +): + configured = { + "memoryDirectory": str(tmp_path / "configured-memory"), + "chromaDbPath": str(tmp_path / "configured-chroma"), + } + monkeypatch.delenv("CHROMA_DB_PATH", raising=False) + monkeypatch.setattr( + handler, + "config_get_by_key", + lambda key, default=None: configured.get(key, default), + ) + + assert handler._resolve_memory_dir() == tmp_path / "configured-memory" + assert handler._resolve_chroma_path() == tmp_path / "configured-chroma" + def test_websocket_defers_memory_export_to_core_dispatch(monkeypatch): config = types.ModuleType("config") From 4a3f1baebebc369d9ff93399e42b02813c435c65 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Thu, 3 Sep 2026 09:34:36 +0300 Subject: [PATCH 31/34] Fix: included version number during export and removed redundant launcher commands --- docs/reference-memory-portability.md | 4 +- scripts/omegaclaw | 35 +++++------- src/memory_export.py | 3 +- tests/test_memory_export.py | 28 ++++++++++ tests/test_omegaclaw_launcher.py | 83 ++++++++++++++++++++++++++++ 5 files changed, 130 insertions(+), 23 deletions(-) create mode 100644 tests/test_omegaclaw_launcher.py diff --git a/docs/reference-memory-portability.md b/docs/reference-memory-portability.md index a4758feb..b92c95a0 100644 --- a/docs/reference-memory-portability.md +++ b/docs/reference-memory-portability.md @@ -63,8 +63,8 @@ scripts/omegaclaw start -d singularitynet/omegaclaw: -p OpenAI -t telegram `overwrite` replaces the selected components after validation and rollback preparation. `append` preserves existing history and adds imported LTM records -under new IDs. Select components with `--only-history`, `--no-history`, or -`--no-vector`. +under new IDs. Select a single component with `--only-history` or +`--only-vector`. Without either option, both components are imported. The importer validates archive paths, checksums, manifest metadata, record counts, and embedding compatibility before changing live memory. It runs before diff --git a/scripts/omegaclaw b/scripts/omegaclaw index 7fdde627..9d1b7b76 100755 --- a/scripts/omegaclaw +++ b/scripts/omegaclaw @@ -554,8 +554,7 @@ help() { echo -e "\t--memory-import restore archive from transfer directory before startup" echo -e "\t--memory-mode overwrite|append import mode (default: overwrite)" echo -e "\t--only-history restore history only" - echo -e "\t--no-history skip history during import" - echo -e "\t--no-vector skip long-term memory during import" + echo -e "\t--only-vector restore long-term memory only" } require_option_value() { @@ -584,8 +583,8 @@ options() { memory_transfer_dir="" memory_import_file="" memory_import_mode="overwrite" - memory_import_no_history=0 - memory_import_no_vector=0 + memory_import_only_history=0 + memory_import_only_vector=0 memory_export_enabled=0 if [[ "$#" -eq 0 ]]; then @@ -637,16 +636,12 @@ options() { memory_import_mode="${2}" shift 2 ;; - --no-history) - memory_import_no_history=1 - shift - ;; - --no-vector) - memory_import_no_vector=1 + --only-history) + memory_import_only_history=1 shift ;; - --only-history) - memory_import_no_vector=1 + --only-vector) + memory_import_only_vector=1 shift ;; --version|-v) version; return 0;; @@ -734,13 +729,13 @@ options() { return 1 fi - if [[ "${memory_import_no_history}" == "1" && "${memory_import_no_vector}" == "1" ]]; then - echo "--no-history cannot be combined with --no-vector or --only-history" >&2 + if [[ "${memory_import_only_history}" == "1" && "${memory_import_only_vector}" == "1" ]]; then + echo "--only-history and --only-vector cannot be combined" >&2 return 1 fi if [[ -z "${memory_import_file}" && - ( "${memory_import_no_history}" == "1" || - "${memory_import_no_vector}" == "1" ) ]]; then + ( "${memory_import_only_history}" == "1" || + "${memory_import_only_vector}" == "1" ) ]]; then echo "Import component flags require --memory-import" >&2 return 1 fi @@ -790,12 +785,12 @@ start() { -e "MEMORY_IMPORT_FILE=${memory_import_file}" -e "MEMORY_IMPORT_MODE=${memory_import_mode}" ) - if [[ "${memory_import_no_history}" == "1" ]]; then - memory_import_env+=(-e MEMORY_IMPORT_NO_HISTORY=1) - fi - if [[ "${memory_import_no_vector}" == "1" ]]; then + if [[ "${memory_import_only_history}" == "1" ]]; then memory_import_env+=(-e MEMORY_IMPORT_NO_VECTOR=1) fi + if [[ "${memory_import_only_vector}" == "1" ]]; then + memory_import_env+=(-e MEMORY_IMPORT_NO_HISTORY=1) + fi fi docker rm -f omegaclaw 2>/dev/null || true diff --git a/src/memory_export.py b/src/memory_export.py index 7e55e7e3..fb4be7fe 100644 --- a/src/memory_export.py +++ b/src/memory_export.py @@ -4,7 +4,7 @@ from pathlib import Path from config import config_get_by_key -from helper import projectRootDirectory +from helper import omegaclaw_version, projectRootDirectory from src.logger import get_logger logger = get_logger(__name__) @@ -54,6 +54,7 @@ def _get_transfer(): raise ValueError(f"Unsupported embedding provider: {embedding_provider!r}") os.environ["EMBEDDING_PROVIDER"] = embedding_provider + os.environ["OMEGACLAW_VERSION"] = omegaclaw_version() _transfer = MemoryTransfer( transfer_dir=_TRANSFER_DIR, store=create_memory_store(), diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index a85383ae..dfdcd14b 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -129,6 +129,34 @@ def __init__(self, **kwargs): }] +def test_transfer_exposes_runtime_omegaclaw_version(handler, monkeypatch): + created = [] + + class FakeTransfer: + def __init__(self, **kwargs): + created.append({ + **kwargs, + "omegaclaw_version": os.environ.get("OMEGACLAW_VERSION"), + }) + + monkeypatch.delenv("OMEGACLAW_VERSION", raising=False) + package = types.ModuleType("memory_portability") + package.MemoryTransfer = FakeTransfer + monkeypatch.setitem(sys.modules, "memory_portability", package) + monkeypatch.setattr(handler, "create_memory_store", lambda: "configured-store") + monkeypatch.setattr(handler, "omegaclaw_version", lambda: "OmegaClaw version=v1.2.3") + handler._transfer = None + + transfer = handler._get_transfer() + + assert transfer is handler._transfer + assert created == [{ + "transfer_dir": handler._TRANSFER_DIR, + "store": "configured-store", + "omegaclaw_version": "OmegaClaw version=v1.2.3", + }] + + def test_memory_store_receives_explicit_omegaclaw_storage_configuration( handler, monkeypatch, diff --git a/tests/test_omegaclaw_launcher.py b/tests/test_omegaclaw_launcher.py new file mode 100644 index 00000000..4022eb7e --- /dev/null +++ b/tests/test_omegaclaw_launcher.py @@ -0,0 +1,83 @@ +import os +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +LAUNCHER = REPO_ROOT / "scripts" / "omegaclaw" + + +def _run_launcher(tmp_path: Path, *component_options: str) -> subprocess.CompletedProcess: + archive = tmp_path / "memory.tar.gz" + archive.touch() + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text( + "#!/bin/sh\n" + "printf 'docker'\n" + "printf ' <%s>' \"$@\"\n" + "printf '\\n'\n", + encoding="utf-8", + ) + docker.chmod(0o755) + + environment = os.environ.copy() + environment["ASI_API_KEY"] = "test-token" + environment["PATH"] = f"{bin_dir}{os.pathsep}{environment['PATH']}" + + return subprocess.run( + [ + str(LAUNCHER), + "start", + "--memory-transfer-dir", + str(tmp_path), + "--memory-import", + archive.name, + *component_options, + ], + cwd=REPO_ROOT, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + +@pytest.mark.parametrize( + ("option", "included_environment", "excluded_environment"), + [ + ("--only-history", "MEMORY_IMPORT_NO_VECTOR=1", "MEMORY_IMPORT_NO_HISTORY=1"), + ("--only-vector", "MEMORY_IMPORT_NO_HISTORY=1", "MEMORY_IMPORT_NO_VECTOR=1"), + ], +) +def test_only_component_options_select_one_import_component( + tmp_path, + option, + included_environment, + excluded_environment, +): + result = _run_launcher(tmp_path, option) + + assert result.returncode == 0, result.stderr + assert included_environment in result.stdout + assert excluded_environment not in result.stdout + + +def test_only_component_options_are_mutually_exclusive(tmp_path): + result = _run_launcher(tmp_path, "--only-history", "--only-vector") + + assert result.returncode != 0 + assert "--only-history and --only-vector cannot be combined" in result.stderr + + +@pytest.mark.parametrize("removed_option", ["--no-history", "--no-vector"]) +def test_removed_component_options_are_rejected(tmp_path, removed_option): + result = _run_launcher(tmp_path, removed_option) + + assert result.returncode != 0 + assert "Usage:" in result.stdout + assert "docker <" not in result.stdout From 60fd8a010429caa6779cab37a0cbe0f06b879798 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Thu, 3 Sep 2026 09:41:40 +0300 Subject: [PATCH 32/34] chore: renamed OMEGACLAW_VERSION to OMEGA_VERSION --- src/memory_export.py | 3 +-- tests/test_memory_export.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/memory_export.py b/src/memory_export.py index fa6fa1f6..f914ee2b 100644 --- a/src/memory_export.py +++ b/src/memory_export.py @@ -54,8 +54,7 @@ def _get_transfer(): raise ValueError(f"Unsupported embedding provider: {embedding_provider!r}") os.environ["EMBEDDING_PROVIDER"] = embedding_provider - # import-kb 0.2.1 uses this legacy metadata key for the source version. - os.environ["OMEGACLAW_VERSION"] = omega_version() + os.environ["OMEGA_VERSION"] = omega_version() _transfer = MemoryTransfer( transfer_dir=_TRANSFER_DIR, store=create_memory_store(), diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index cfe3ca16..c8c57017 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -136,10 +136,10 @@ class FakeTransfer: def __init__(self, **kwargs): created.append({ **kwargs, - "omegaclaw_version": os.environ.get("OMEGACLAW_VERSION"), + "omega_version": os.environ.get("OMEGA_VERSION"), }) - monkeypatch.delenv("OMEGACLAW_VERSION", raising=False) + monkeypatch.delenv("OMEGA_VERSION", raising=False) package = types.ModuleType("memory_portability") package.MemoryTransfer = FakeTransfer monkeypatch.setitem(sys.modules, "memory_portability", package) @@ -153,7 +153,7 @@ def __init__(self, **kwargs): assert created == [{ "transfer_dir": handler._TRANSFER_DIR, "store": "configured-store", - "omegaclaw_version": "Omega version=v1.2.3", + "omega_version": "Omega version=v1.2.3", }] From 6d43ea5a5fae3a7bc4e1fb32956fb38cf3acbd9b Mon Sep 17 00:00:00 2001 From: CodersKin Date: Thu, 3 Sep 2026 12:38:28 +0300 Subject: [PATCH 33/34] chore: updated import-kb version number --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d819dcb0..45e17674 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ chromadb==1.5.9 openai==2.38.0 transformers==5.8.0 sentence-transformers==5.5.1 -import-kb==0.2.1 +import-kb==0.2.3 py-landlock==0.1.1 pyyaml==6.0.3 ddgs==9.14.4 From 15a1c5db9096df9e846ae623351ccccc38110171 Mon Sep 17 00:00:00 2001 From: CodersKin Date: Thu, 3 Sep 2026 13:02:25 +0300 Subject: [PATCH 34/34] Fix memory portability startup with import-kb 0.2.3 --- entrypoint.sh | 5 ++++- tests/test_memory_export.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index c58497bb..a89df4dc 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -46,7 +46,10 @@ from memory_export import create_memory_store from memory_portability import MemoryTransfer init_config([]) -transfer = MemoryTransfer(store=create_memory_store()) +transfer = MemoryTransfer( + transfer_dir="/memory-transfer", + store=create_memory_store(), +) operation = os.environ["MEMORY_PORTABILITY_OPERATION"] if operation == "recover": transfer.recover() diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py index c8c57017..feedf891 100644 --- a/tests/test_memory_export.py +++ b/tests/test_memory_export.py @@ -64,6 +64,12 @@ def test_module_import_does_not_require_memory_portability(handler): assert "memory_portability" not in sys.modules assert handler.is_export_command("/memory-export both") + +def test_entrypoint_configures_memory_transfer_directory(): + entrypoint = (REPO_ROOT / "entrypoint.sh").read_text(encoding="utf-8") + + assert 'transfer_dir="/memory-transfer"' in entrypoint + @pytest.mark.parametrize("component", ["history", "ltm", "both"]) def test_export_runs_immediately(handler, component): exported = []