diff --git a/Dockerfile b/Dockerfile index ded48a6f..e0766618 100644 --- a/Dockerfile +++ b/Dockerfile @@ -134,11 +134,14 @@ COPY --from=versioned-source /omega-source ${OMEGA_DIR} RUN cp ${OMEGA_DIR}/run.metta /PeTTa/run.metta \ && mkdir -p ${MEMORY_DIR}/chroma_db \ + && mkdir -p /memory-transfer \ && ln -s ${MEMORY_DIR}/chroma_db ./chroma_db \ && chmod +x ${OMEGA_DIR}/entrypoint.sh \ && chmod +x ${OMEGA_DIR}/scripts/import_knowledge.sh \ && chmod +x ${OMEGA_DIR}/scripts/omega \ && chown -R 65534:65534 ${MEMORY_DIR} \ + && chown 65534:65534 /memory-transfer \ + && chmod 0700 /memory-transfer \ && find ${MEMORY_DIR} -type f -exec chmod 0644 {} \; \ && chmod 0444 ${MEMORY_DIR}/prompt.txt \ && chown -R 65534:65534 /opt/huggingface /opt/sentence_transformers diff --git a/README.md b/README.md index 60d64669..7fa466f9 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,25 @@ To reset Omega's memory: docker volume rm omega-memory ``` +### Memory portability + +Memory export is disabled by default. See the [memory portability reference](./docs/reference-memory-portability.md) +for setup, export controls, archive contents, and import modes. + +> **Current limitation:** Memory import does not work in a standalone Omega +> run. Import and interrupted-import recovery are supported only through Docker +> using `scripts/omega`, because both operations run from the container +> entrypoint before the agent loop starts. + +To restore an archive while upgrading to a tagged image, use the same transfer directory: + +```sh +scripts/omega start -d singularitynet/omega: -p OpenAI -t telegram \ + --memory-transfer-dir "$HOME/omega-transfers" \ + --memory-import omegaclaw-memory-.tar.gz \ + --memory-mode overwrite +``` + --- ## Usage diff --git a/config/config.yaml b/config/config.yaml index 6a81ae94..9d74a649 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -21,8 +21,8 @@ wakeupInterval: 600 # Path to the logger configuration file # See https://docs.python.org/3/library/logging.config.html#configuration-file-format logConfigPath: "" -# (internal) Path to the memory directory -#memoryDirectory: "./" +# Directory containing persistent memory files such as history.metta. +memoryDirectory: "./repos/Omega/memory" # Memory @@ -34,8 +34,12 @@ maxRecallItems: 20 maxEpisodeRecallLines: 20 # Tail of `memory/history.metta` included in the prompt (chars) maxHistory: 30000 +# ChromaDB persistence directory used for long-term memory portability. +chromaDbPath: "./chroma_db" # `Local` (Python-side model) or `OpenAI` (requires `OPENAI_API_KEY`) embeddingprovider: Local +# Enable authenticated operator-triggered /memory-export commands (disabled by default). +memoryExportEnabled: false # Policy diff --git a/docs/README.md b/docs/README.md index 15dc3d5d..ed63d60e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,6 +57,7 @@ User-facing MeTTa skills the agent invokes. Each page follows the template **Sig - [reference-configuration.md](./reference-configuration.md) — `configure` form and all runtime parameters - [reference-channels.md](./reference-channels.md) — IRC, Telegram, Slack, Mattermost, WebSocket, and websearch adapters plus the channel contract - [reference-python-bridges.md](./reference-python-bridges.md) — `lib_llm_ext.py`, `src/helper.py`, `src/skills.pl` +- [reference-memory-portability.md](./reference-memory-portability.md) — Operator backup, restore, and archive-transfer workflow ### Plugin API diff --git a/docs/reference-channels.md b/docs/reference-channels.md index 3f15fde5..03513ef1 100644 --- a/docs/reference-channels.md +++ b/docs/reference-channels.md @@ -79,6 +79,7 @@ Minimal JSON chat adapter over a WebSocket connection. Selected with `commchanne - `stop_websocket()` — stop the listener thread and close the socket. - Requires the `websockets` Python package. - When `WS_TOKEN` is set it is sent as an `Authorization: Bearer ` header. Unlike the IRC/Telegram/Slack adapters there is no one-time `auth ` gate — trust is established by the endpoint URL and bearer token. +- Supports immediate `/memory-export history|ltm|both` commands when memory export is enabled and `WS_TOKEN` is configured. The export handler uses a SHA-256-derived connection principal and never exposes the bearer token. Protect the endpoint with `wss://` and server-side access controls because WebSocket does not have the per-user ownership gate used by the other channels. - Reconnects automatically with exponential backoff (1s → 30s, ±20% jitter) and is safe to start once at process startup. ### Frame protocol diff --git a/docs/reference-configuration.md b/docs/reference-configuration.md index 2cc28f2c..0b9c6262 100644 --- a/docs/reference-configuration.md +++ b/docs/reference-configuration.md @@ -31,6 +31,8 @@ This reads a command-line override via `argk` (`name=value` on the MeTTa command | `maxRecallItems` | 20 | Items returned by `query`. | | `maxEpisodeRecallLines` | 20 | Lines returned by `episodes`. | | `maxHistory` | 30000 (chars) | Tail of `memory/history.metta` included in the prompt. | +| `memoryDirectory` | `./repos/Omega/memory` | Directory containing persistent memory files such as `history.metta`. | +| `chromaDbPath` | `./chroma_db` | ChromaDB persistence directory used for memory backup and restore. | | `embeddingprovider` | `Local` | `Local` (Python-side model) or `OpenAI`. | ## Channels (`src/channels.metta`, `initChannels`) @@ -68,7 +70,7 @@ metta run.metta provider=Anthropic LLM=claude-opus-4-6 commchannel=mattermost ``` Configuration values are resolved in this order: command-line `key=value`, -`OMEGACLAW_` environment variable, `config/config.yaml`, then the caller's +`OMEGA_` environment variable, `config/config.yaml`, then the caller's default. `TG_BOT_TOKEN` and `OMEGA_AUTH_SECRET` are read directly from the environment and must be placed before the `metta`/`petta` command. diff --git a/docs/reference-memory-portability.md b/docs/reference-memory-portability.md new file mode 100644 index 00000000..0cc87b4d --- /dev/null +++ b/docs/reference-memory-portability.md @@ -0,0 +1,78 @@ +# Reference - Memory Portability + +Memory portability lets an operator export persistent user memory from one +deployment and restore it before another agent starts. It is an operator +workflow, not an LLM skill. + +## Setup + +Choose an absolute host directory for archives. The launcher creates it when +needed and mounts it at the fixed container path `/memory-transfer`; the agent +never accepts arbitrary runtime export paths. + +```sh +scripts/omega start -p OpenAI -t telegram \ + --memory-transfer-dir "$HOME/omega-transfers" \ + --enable-memory-export +``` + +`--enable-memory-export` is required because export is disabled by default. +The transfer directory must be writable by the container's agent user. + +## Export + +In the active chat, request one component. The export runs immediately. For IRC, +Telegram, Slack, and Mattermost, the export handler requires the authenticated +user ID persisted by the channel authorization layer. WebSocket export requires +a configured `WS_TOKEN`; the handler derives a non-reversible principal from the +token so the credential itself is never used as an identifier: + +```text +/memory-export history +/memory-export ltm +/memory-export both +``` + +Completion is delivered through the active channel and includes the filename, +record count, size, and SHA-256. + +Archives contain selected persistent user memory only: + +```text +manifest.json +history/history.metta +vector/collections.json +vector/records.jsonl +``` + +History is the conversation trace. LTM is logical user-memory records from +ChromaDB. Prompts, credentials, logs, skills, and other operational state are +not exported. SHA-256 detects corruption, not archive authorship. + +## Import + +Import is an administrative startup operation. The archive argument is a plain +filename in the chosen transfer directory: + +```sh +scripts/omega start -d singularitynet/omega: -p OpenAI -t telegram \ + --memory-transfer-dir "$HOME/omega-transfers" \ + --memory-import omegaclaw-memory-.tar.gz \ + --memory-mode overwrite +``` + +`overwrite` replaces the selected components after validation and rollback +preparation. `append` preserves existing history and adds imported LTM records +under new IDs. Select a single component with `--only-history` or +`--only-vector`. Without either option, both components are imported. + +The importer validates archive paths, checksums, manifest metadata, record +counts, and embedding compatibility before changing live memory. It runs before +the agent loop starts. A receipt prevents a completed archive import from +running again on container restart. + +## Security + +Archives are private operator data; keep the host transfer directory protected. +Memory export is denied when channel authentication is disabled, no authenticated +channel user has been persisted, or WebSocket has no `WS_TOKEN`. diff --git a/entrypoint.sh b/entrypoint.sh index 50d6cd44..a89df4dc 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -40,11 +40,49 @@ if [[ "${IMPORT_KB_ON_START}" == "1" ]]; then su nobody -s /bin/sh -c "${OMEGA_DIR}/scripts/import_knowledge.sh" fi +MEMORY_PORTABILITY_PYTHON='import os +from config import init_config +from memory_export import create_memory_store +from memory_portability import MemoryTransfer + +init_config([]) +transfer = MemoryTransfer( + transfer_dir="/memory-transfer", + store=create_memory_store(), +) +operation = os.environ["MEMORY_PORTABILITY_OPERATION"] +if operation == "recover": + transfer.recover() +elif operation == "import": + transfer.import_archive( + os.environ["MEMORY_IMPORT_FILE"], + mode=os.environ.get("MEMORY_IMPORT_MODE", "overwrite"), + include_history=os.environ.get("MEMORY_IMPORT_NO_HISTORY") != "1", + include_vectors=os.environ.get("MEMORY_IMPORT_NO_VECTOR") != "1", + ) +else: + raise ValueError(f"Unsupported memory portability operation: {operation!r}")' +export MEMORY_PORTABILITY_PYTHON +export PYTHONPATH="${OMEGA_DIR}:${OMEGA_DIR}/src${PYTHONPATH:+:${PYTHONPATH}}" + +export MEMORY_PORTABILITY_OPERATION=recover +su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \ + || { echo "Memory import recovery failed. Aborting startup." >&2; exit 1; } + +if [[ -n "${MEMORY_IMPORT_FILE:-}" ]]; then + echo "memory_portability: importing ${MEMORY_IMPORT_FILE}" + export MEMORY_PORTABILITY_OPERATION=import + su nobody -s /bin/sh -c 'exec python3 -c "$MEMORY_PORTABILITY_PYTHON"' \ + || { echo "Memory import failed. Aborting startup." >&2; exit 1; } + echo "memory_portability: import complete" +fi +unset MEMORY_PORTABILITY_OPERATION MEMORY_PORTABILITY_PYTHON PYTHONPATH + # Scrub environment: only allowlisted vars survive. SAFE_VARS="HOME USER PATH HOSTNAME TERM LANG LC_ALL \ PYTHONDONTWRITEBYTECODE PYTHONUNBUFFERED \ HF_HOME SENTENCE_TRANSFORMERS_HOME HF_HUB_OFFLINE TRANSFORMERS_OFFLINE \ - OMEGA_DIR MEMORY_DIR TEST_SERVER_IP" + CHROMA_DB_PATH EMBEDDING_PROVIDER OMEGA_DIR MEMORY_DIR TEST_SERVER_IP" env_args="" for var in $SAFE_VARS; do diff --git a/profile/policy.yaml b/profile/policy.yaml index 840d58d3..22c3d240 100644 --- a/profile/policy.yaml +++ b/profile/policy.yaml @@ -14,6 +14,7 @@ filesystem_policy: - /PeTTa read_write: - /PeTTa/repos/Omega/memory + - /memory-transfer - /tmp - /dev/null - /opt/huggingface diff --git a/requirements.txt b/requirements.txt index 785be39a..45e17674 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ chromadb==1.5.9 openai==2.38.0 transformers==5.8.0 sentence-transformers==5.5.1 -import-kb==0.1.8 +import-kb==0.2.3 py-landlock==0.1.1 pyyaml==6.0.3 ddgs==9.14.4 diff --git a/scripts/omega b/scripts/omega index eeb7bd9b..bcd67b7b 100755 --- a/scripts/omega +++ b/scripts/omega @@ -14,6 +14,7 @@ chmod 0600 "$tmp_py_file" cat >"$tmp_py_file" <<'PY' import getpass import json +import os import secrets import string import shlex @@ -282,6 +283,42 @@ def _choose_import_kb(): print("Please enter y or n.") +def _choose_memory_transfer(): + while True: + answer = input("Set up a directory for memory transfer archives? [y/N]: ").strip().lower() + if answer in ("", "n", "no"): + return "", "0" + if answer == "q": + sys.exit(1) + if answer not in ("y", "yes"): + print("Please enter y or n.") + continue + + value = input("Host directory for memory archives: ").strip() + if not value: + print("A directory is required.", file=sys.stderr) + continue + path = os.path.abspath(os.path.expanduser(value)) + try: + os.makedirs(path, exist_ok=True) + except OSError as exc: + print(f"Could not create {path}: {exc}", file=sys.stderr) + continue + if not os.access(path, os.W_OK): + print(f"Directory is not writable: {path}", file=sys.stderr) + continue + + while True: + enable = input("Enable memory export for this instance? [y/N]: ").strip().lower() + if enable in ("", "n", "no"): + return path, "0" + if enable in ("y", "yes"): + return path, "1" + if enable == "q": + sys.exit(1) + print("Please enter y or n.") + + def _prompt_llm_token(): while True: token = getpass.getpass("Please paste your LLM token and press ENTER or 'q' to exit: ").strip() @@ -300,7 +337,11 @@ def _write_kv(fp, key, value): fp.write(f"{key}={shlex.quote(str(value))}\n") -def config_run_omega(config_output_path): +def config_run_omega( + config_output_path, + configured_memory_transfer_dir="", + configured_memory_export_enabled="0", +): print(" ") print("Welcome to Omega!") print(" ") @@ -309,6 +350,12 @@ def config_run_omega(config_output_path): channel_config = _choose_channel() provider, embeddingprovider, api_token_var, model, openaiapi_url, token = _choose_provider() import_kb_on_start = _choose_import_kb() + if configured_memory_transfer_dir: + memory_transfer_dir = configured_memory_transfer_dir + memory_export_enabled = configured_memory_export_enabled + print(f"Using memory transfer directory: {memory_transfer_dir}") + else: + memory_transfer_dir, memory_export_enabled = _choose_memory_transfer() with open(config_output_path, "w", encoding="utf-8") as f: _write_kv(f, "api_token_var", api_token_var) @@ -319,15 +366,18 @@ def config_run_omega(config_output_path): _write_kv(f, "model", model) _write_kv(f, "openaiapi_url", openaiapi_url) _write_kv(f, "IMPORT_KB_ON_START", import_kb_on_start) + _write_kv(f, "memory_transfer_dir", memory_transfer_dir) + _write_kv(f, "memory_export_enabled", memory_export_enabled) for key, value in channel_config.items(): _write_kv(f, key, value) if __name__ == "__main__": - config_run_omega(sys.argv[1]) + config_run_omega(*sys.argv[1:4]) PY -python3 "$tmp_py_file" "$tmp_config_file" set Python logging config file" echo -e "\t--version, -v show the Omega version" echo -e "\t--help, -h show this help" + echo + echo -e "Memory portability options:" + echo -e "\t--memory-transfer-dir mount host directory for memory export/import archives" + echo -e "\t--enable-memory-export enable memory export" + echo -e "\t--memory-import restore archive from transfer directory before startup" + echo -e "\t--memory-mode overwrite|append import mode (default: overwrite)" + echo -e "\t--only-history restore history only" + echo -e "\t--only-vector restore long-term memory only" +} + +require_option_value() { + if [[ "$#" -lt 2 || -z "${2}" ]]; then + echo "${1} requires a value" >&2 + return 1 + fi } options() { @@ -515,6 +580,12 @@ options() { OMEGA_OPENCLAW_TOKEN="${OMEGA_OPENCLAW_TOKEN:-}" commchannel=irc log_config_path="" + memory_transfer_dir="" + memory_import_file="" + memory_import_mode="overwrite" + memory_import_only_history=0 + memory_import_only_vector=0 + memory_export_enabled=0 if [[ "$#" -eq 0 ]]; then help @@ -524,18 +595,57 @@ options() { command=${1} shift 1 - while getopts s:t:p:u:g:c:d:l:m: flag - do - case "${flag}" in - s) OMEGA_AUTH_SECRET=${OPTARG};; - t) commchannel=${OPTARG};; - p) provider=${OPTARG};; - u) openaiapi_url=${OPTARG};; - g) openclaw_url=${OPTARG};; - c) IRC_channel=${OPTARG};; - d) image=${OPTARG};; - l) log_config_path=${OPTARG};; - m) model=${OPTARG};; + while [[ "$#" -gt 0 ]]; do + case "${1}" in + -s?*) OMEGA_AUTH_SECRET=${1:2}; shift; continue;; + -t?*) commchannel=${1:2}; shift; continue;; + -p?*) provider=${1:2}; shift; continue;; + -u?*) openaiapi_url=${1:2}; shift; continue;; + -g?*) openclaw_url=${1:2}; shift; continue;; + -c?*) IRC_channel=${1:2}; shift; continue;; + -d?*) image=${1:2}; shift; continue;; + -l?*) log_config_path=${1:2}; shift; continue;; + -m?*) model=${1:2}; shift; continue;; + -s|-t|-p|-u|-g|-c|-d|-l|-m|--memory-transfer-dir|--memory-import|--memory-mode) + require_option_value "${1}" "${2:-}" || return 1 + ;; + esac + case "${1}" in + -s) OMEGA_AUTH_SECRET=${2}; shift 2;; + -t) commchannel=${2}; shift 2;; + -p) provider=${2}; shift 2;; + -u) openaiapi_url=${2}; shift 2;; + -g) openclaw_url=${2}; shift 2;; + -c) IRC_channel=${2}; shift 2;; + -d) image=${2}; shift 2;; + -l) log_config_path=${2}; shift 2;; + -m) model=${2}; shift 2;; + --memory-transfer-dir) + memory_transfer_dir="${2}" + shift 2 + ;; + --enable-memory-export) + memory_export_enabled=1 + shift + ;; + --memory-import) + memory_import_file="${2}" + shift 2 + ;; + --memory-mode) + memory_import_mode="${2}" + shift 2 + ;; + --only-history) + memory_import_only_history=1 + shift + ;; + --only-vector) + memory_import_only_vector=1 + shift + ;; + --version|-v) version; return 0;; + --help|-h) help; return 0;; *) help return 1 ;; @@ -583,14 +693,67 @@ options() { return 1 fi fi + + if [[ -n "${memory_transfer_dir}" ]]; then + if [[ "${memory_transfer_dir}" != /* ]]; then + echo "--memory-transfer-dir must be an absolute path: ${memory_transfer_dir}" >&2 + return 1 + fi + if ! mkdir -p "${memory_transfer_dir}"; then + echo "Could not create --memory-transfer-dir: ${memory_transfer_dir}" >&2 + return 1 + fi + if [[ ! -w "${memory_transfer_dir}" ]]; then + echo "--memory-transfer-dir is not writable: ${memory_transfer_dir}" >&2 + return 1 + fi + fi + + if [[ -n "${memory_import_file}" ]]; then + if [[ ! "${memory_import_file}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*\.tar\.gz$ ]]; then + echo "--memory-import must be a plain filename, not a path: ${memory_import_file}" >&2 + return 1 + fi + if [[ -z "${memory_transfer_dir}" ]]; then + echo "--memory-import requires --memory-transfer-dir to be set" >&2 + return 1 + fi + if [[ ! -f "${memory_transfer_dir}/${memory_import_file}" ]]; then + echo "--memory-import archive not found: ${memory_transfer_dir}/${memory_import_file}" >&2 + return 1 + fi + fi + + if [[ "${memory_export_enabled}" == "1" && -z "${memory_transfer_dir}" ]]; then + echo "--enable-memory-export requires --memory-transfer-dir" >&2 + return 1 + fi + + if [[ "${memory_import_only_history}" == "1" && "${memory_import_only_vector}" == "1" ]]; then + echo "--only-history and --only-vector cannot be combined" >&2 + return 1 + fi + if [[ -z "${memory_import_file}" && + ( "${memory_import_only_history}" == "1" || + "${memory_import_only_vector}" == "1" ) ]]; then + echo "Import component flags require --memory-import" >&2 + return 1 + fi + + if [[ "${memory_import_mode}" != "overwrite" && "${memory_import_mode}" != "append" ]]; then + echo "--memory-mode must be overwrite or append" >&2 + return 1 + fi } start() { - docker rm -f omega 2>/dev/null || true docker pull "${image}" 2>/dev/null || true container_log_config_path="" log_config_volume=() + memory_transfer_volume=() + memory_import_env=() + memory_export_option=() if [ -n "${log_config_path}" ]; then if [ ! -f "${log_config_path}" ]; then @@ -603,6 +766,35 @@ start() { log_config_volume=(--volume "${log_config_abs}:${container_log_config_path}:ro") fi + if [ -n "${memory_transfer_dir}" ]; then + memory_transfer_volume=(--volume "${memory_transfer_dir}:/memory-transfer") + if ! docker run --rm --user 65534:65534 --entrypoint /bin/sh \ + --volume "${memory_transfer_dir}:/memory-transfer" "${image}" \ + -c 'test -d /memory-transfer && test -w /memory-transfer'; then + echo "--memory-transfer-dir must be writable by container user 65534:65534" >&2 + return 1 + fi + fi + + if [[ "${memory_export_enabled}" == "1" ]]; then + memory_export_option=("memoryExportEnabled=true") + fi + + if [ -n "${memory_import_file}" ]; then + memory_import_env=( + -e "MEMORY_IMPORT_FILE=${memory_import_file}" + -e "MEMORY_IMPORT_MODE=${memory_import_mode}" + ) + if [[ "${memory_import_only_history}" == "1" ]]; then + memory_import_env+=(-e MEMORY_IMPORT_NO_VECTOR=1) + fi + if [[ "${memory_import_only_vector}" == "1" ]]; then + memory_import_env+=(-e MEMORY_IMPORT_NO_HISTORY=1) + fi + fi + + docker rm -f omega 2>/dev/null || true + docker_cmd=( docker run -d -it --name omega @@ -614,17 +806,20 @@ start() { --tmpfs /run:size=16m,mode=755 --volume omega-memory:/PeTTa/repos/Omega/memory ${log_config_volume[@]+"${log_config_volume[@]}"} + ${memory_transfer_volume[@]+"${memory_transfer_volume[@]}"} -e "${api_token_var}"="${api_token}" -e "TG_BOT_TOKEN=${TG_BOT_TOKEN:-}" -e "SL_BOT_TOKEN=${SL_BOT_TOKEN:-}" -e "OMEGA_OPENCLAW_TOKEN=${OMEGA_OPENCLAW_TOKEN:-}" -e OMEGA_AUTH_SECRET="$OMEGA_AUTH_SECRET" -e IMPORT_KB_ON_START="${IMPORT_KB_ON_START:-0}" + ${memory_import_env[@]+"${memory_import_env[@]}"} "$image" "commchannel=${commchannel}" "provider=${provider}" "embeddingprovider=${embeddingprovider}" "securityPolicyPath=/PeTTa/repos/Omega/profile/policy.yaml" + ${memory_export_option[@]+"${memory_export_option[@]}"} ) if [ -n "${container_log_config_path}" ]; then diff --git a/src/channels.py b/src/channels.py index 222bab85..78f312a3 100644 --- a/src/channels.py +++ b/src/channels.py @@ -1,9 +1,52 @@ +import hashlib import logging logger = logging.getLogger(__name__) _commChannelRegistry = {} + +def _authenticated_export_principal() -> str | None: + if _commchannel_id == "websocket": + from config import config_get_by_key + + token = str(config_get_by_key("WS_TOKEN", "")).strip() + if not token: + return None + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return f"websocket:{digest}" + + from auth import get_channel_authenticated_user_id, is_auth_enabled + + if not is_auth_enabled(): + return None + return get_channel_authenticated_user_id(_commchannel_id.upper()) + + +def handle_control_message(message: str) -> bool: + from src.memory_export import handle_export_command, is_export_command + + _, separator, command = message.rpartition(": ") + if not separator: + command = message + if not is_export_command(command): + return False + + try: + authenticated_principal = _authenticated_export_principal() + except Exception as exc: + logger.exception("Failed to resolve memory-export principal: %s", exc) + authenticated_principal = None + + reply = handle_export_command(command, authenticated_principal) + if reply is not None: + try: + _commchannel.send(reply) + except Exception as exc: + logger.exception("Failed to deliver control-message response: %s", exc) + return True + + class CommChannel: """Communication channel implementation""" @@ -36,22 +79,27 @@ def registerCommChannel(id: str, channel: CommChannel) -> None: _commChannelRegistry[id] = channel _commchannel: CommChannel = None +_commchannel_id = "" def commChannelStart(commchannel): """Select and start one of the communication channels registered by plugins""" - global _commchannel + global _commchannel, _commchannel_id _commchannel = _commChannelRegistry.get(commchannel, None) if _commchannel is None: error = f"commChannelStart: Communication channel plugin {commchannel} is not registered" logger.error(error) raise RuntimeError(error) + _commchannel_id = str(commchannel).lower() _commchannel.start() def commChannelReceive(): """Receive message from selected communication channel""" global _commchannel - return _commchannel.receive() + messages = _commchannel.receive().split(" | ") + return " | ".join( + message for message in messages if not handle_control_message(message) + ) def commChannelSend(message): """Send message via selected communication channel""" diff --git a/src/memory_export.py b/src/memory_export.py new file mode 100644 index 00000000..f914ee2b --- /dev/null +++ b/src/memory_export.py @@ -0,0 +1,131 @@ +"""Shared /memory-export command handling.""" + +import os +from pathlib import Path + +from config import config_get_by_key +from helper import omega_version, projectRootDirectory +from src.logger import get_logger + +logger = get_logger(__name__) + +_TRANSFER_DIR = Path("/memory-transfer") + +_transfer = None + + +def _resolve_memory_dir() -> Path: + default = os.environ.get( + "MEMORY_DIR", + str(Path(projectRootDirectory()) / "memory"), + ) + configured = config_get_by_key("memoryDirectory", default) + return Path(str(configured)).expanduser().resolve() + + +def _resolve_chroma_path() -> Path: + environment_path = os.environ.get("CHROMA_DB_PATH") + if environment_path: + return Path(environment_path).expanduser().resolve() + + default = str(Path(projectRootDirectory()).parents[1] / "chroma_db") + configured = config_get_by_key("chromaDbPath", default) + return Path(str(configured)).expanduser().resolve() + + +def create_memory_store(): + """Build an import-kb store from Omega's effective configuration.""" + from memory_portability.storage import MemoryStore + + return MemoryStore( + memory_dir=_resolve_memory_dir(), + chroma_path=_resolve_chroma_path(), + collection_name="memories", + ) + + +def _get_transfer(): + global _transfer + if _transfer is None: + from memory_portability import MemoryTransfer + + embedding_provider = str(config_get_by_key("embeddingprovider", "Local")).strip() + if embedding_provider.casefold() not in {"local", "openai"}: + raise ValueError(f"Unsupported embedding provider: {embedding_provider!r}") + + os.environ["EMBEDDING_PROVIDER"] = embedding_provider + os.environ["OMEGA_VERSION"] = omega_version() + _transfer = MemoryTransfer( + transfer_dir=_TRANSFER_DIR, + store=create_memory_store(), + ) + return _transfer + + +_VALID_COMPONENTS = ("history", "ltm", "both") + + +def is_export_enabled() -> bool: + value = config_get_by_key("memoryExportEnabled", False) + return value is True or ( + isinstance(value, str) and value.strip().lower() == "true" + ) + + +def is_export_command(text: str) -> bool: + command = text.strip().split(None, 1) + if not command: + return False + name = command[0].lower() + return name == "/memory-export" or name.startswith("/memory-export@") + + +def _command_arguments(text: str) -> str: + parts = text.strip().split(None, 1) + return parts[1].strip() if len(parts) == 2 else "" + + +def handle_export_command( + text: str, + authenticated_user_id: str | None = None, +) -> str | None: + stripped = text.strip() + + if not is_export_command(stripped): + return None + + if not is_export_enabled(): + return None + + if not authenticated_user_id: + return "Memory export denied: an authenticated user is required." + + rest = _command_arguments(stripped) + component = rest.lower() + + if component in _VALID_COMPONENTS: + return _export(component) + + return ( + "Unknown /memory-export command. " + "Use: /memory-export history|ltm|both" + ) + + +def _export(component: str) -> str: + try: + result = _get_transfer().export(component) + return _format_export(result) + except Exception as exc: + logger.exception(f"memory_export: export failed: {exc}") + return f"Memory export failed: {exc}" + + +def _format_export(result: dict) -> str: + return ( + "Memory export complete\n" + f"File: {result.get('filename')}\n" + f"Size: {result.get('size')} bytes\n" + f"SHA-256: {result.get('sha256', result.get('checksum'))}\n" + f"Records: {result.get('record_count')}" + ) diff --git a/tests/test_memory_export.py b/tests/test_memory_export.py new file mode 100644 index 00000000..feedf891 --- /dev/null +++ b/tests/test_memory_export.py @@ -0,0 +1,381 @@ +import importlib +import importlib.util +import hashlib +import json +import os +import sys +import types +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] + +@pytest.fixture +def handler(monkeypatch): + logger_mod = types.ModuleType("src.logger") + logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) + monkeypatch.setitem(sys.modules, "src.logger", logger_mod) + + monkeypatch.delitem(sys.modules, "memory_portability", raising=False) + + spec = importlib.util.spec_from_file_location( + "memory_export_under_test", + REPO_ROOT / "src" / "memory_export.py", + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + module.is_export_enabled = lambda: True + return module + +def test_export_command_requires_policy(handler): + exported = [] + handler._get_transfer = lambda: types.SimpleNamespace( + export=lambda component: exported.append(component) or { + "filename": "memory.tar.gz", + "size": 1, + "sha256": "abc", + "record_count": 1, + } + ) + + assert "Memory export complete" in handler.handle_export_command( + "/memory-export both", "authenticated-user" + ) + assert exported == ["both"] + + handler.is_export_enabled = lambda: False + assert handler.handle_export_command( + "/memory-export both", "authenticated-user" + ) is None + assert exported == ["both"] + + +def test_export_requires_authenticated_user(handler): + handler._get_transfer = lambda: pytest.fail( + "an unauthenticated command must not start an export" + ) + + assert handler.handle_export_command("/memory-export both") == ( + "Memory export denied: an authenticated user is required." + ) + +def test_module_import_does_not_require_memory_portability(handler): + assert "memory_portability" not in sys.modules + assert handler.is_export_command("/memory-export both") + + +def test_entrypoint_configures_memory_transfer_directory(): + entrypoint = (REPO_ROOT / "entrypoint.sh").read_text(encoding="utf-8") + + assert 'transfer_dir="/memory-transfer"' in entrypoint + +@pytest.mark.parametrize("component", ["history", "ltm", "both"]) +def test_export_runs_immediately(handler, component): + exported = [] + handler._get_transfer = lambda: types.SimpleNamespace( + export=lambda component: exported.append(component) or { + "filename": "memory.tar.gz", + "size": 1, + "sha256": "abc", + "record_count": 1, + } + ) + reply = handler.handle_export_command( + f"/memory-export {component}", "authenticated-user" + ) + assert exported == [component] + assert "memory.tar.gz" in reply + assert "SHA-256: abc" in reply + + +def test_confirmation_command_is_no_longer_supported(handler): + handler._get_transfer = lambda: pytest.fail( + "the removed confirmation command must not start an export" + ) + + reply = handler.handle_export_command( + "/memory-export confirm old-token", "authenticated-user" + ) + + assert reply == ( + "Unknown /memory-export command. " + "Use: /memory-export history|ltm|both" + ) + +def test_transfer_uses_effective_runtime_embedding_provider(handler, monkeypatch): + created = [] + + class FakeTransfer: + def __init__(self, **kwargs): + created.append({ + **kwargs, + "embedding_provider": os.environ["EMBEDDING_PROVIDER"], + }) + + monkeypatch.delenv("EMBEDDING_PROVIDER", raising=False) + package = types.ModuleType("memory_portability") + package.MemoryTransfer = FakeTransfer + monkeypatch.setitem(sys.modules, "memory_portability", package) + monkeypatch.setattr(handler, "create_memory_store", lambda: "configured-store") + monkeypatch.setattr( + handler, + "config_get_by_key", + lambda key, default=None: "OpenAI" if key == "embeddingprovider" else default, + ) + handler._transfer = None + + transfer = handler._get_transfer() + + assert transfer is handler._transfer + assert created == [{ + "transfer_dir": handler._TRANSFER_DIR, + "store": "configured-store", + "embedding_provider": "OpenAI", + }] + + +def test_transfer_exposes_runtime_omega_version(handler, monkeypatch): + created = [] + + class FakeTransfer: + def __init__(self, **kwargs): + created.append({ + **kwargs, + "omega_version": os.environ.get("OMEGA_VERSION"), + }) + + monkeypatch.delenv("OMEGA_VERSION", raising=False) + package = types.ModuleType("memory_portability") + package.MemoryTransfer = FakeTransfer + monkeypatch.setitem(sys.modules, "memory_portability", package) + monkeypatch.setattr(handler, "create_memory_store", lambda: "configured-store") + monkeypatch.setattr(handler, "omega_version", lambda: "Omega version=v1.2.3") + handler._transfer = None + + transfer = handler._get_transfer() + + assert transfer is handler._transfer + assert created == [{ + "transfer_dir": handler._TRANSFER_DIR, + "store": "configured-store", + "omega_version": "Omega version=v1.2.3", + }] + + +def test_memory_store_receives_explicit_omega_storage_configuration( + handler, + monkeypatch, + tmp_path, +): + created_stores = [] + + class FakeStore: + def __init__(self, **kwargs): + created_stores.append(kwargs) + + package = types.ModuleType("memory_portability") + package.__path__ = [] + storage = types.ModuleType("memory_portability.storage") + storage.MemoryStore = FakeStore + monkeypatch.setitem(sys.modules, "memory_portability", package) + monkeypatch.setitem(sys.modules, "memory_portability.storage", storage) + + memory_dir = tmp_path / "custom-memory" + chroma_path = tmp_path / "custom-chroma" + monkeypatch.setattr(handler, "_resolve_memory_dir", lambda: memory_dir) + monkeypatch.setattr(handler, "_resolve_chroma_path", lambda: chroma_path) + + store = handler.create_memory_store() + + assert isinstance(store, FakeStore) + assert created_stores == [ + { + "memory_dir": memory_dir, + "chroma_path": chroma_path, + "collection_name": "memories", + } + ] + + +def test_storage_paths_are_resolved_from_omega_config( + handler, + monkeypatch, + tmp_path, +): + configured = { + "memoryDirectory": str(tmp_path / "configured-memory"), + "chromaDbPath": str(tmp_path / "configured-chroma"), + } + monkeypatch.delenv("CHROMA_DB_PATH", raising=False) + monkeypatch.setattr( + handler, + "config_get_by_key", + lambda key, default=None: configured.get(key, default), + ) + + assert handler._resolve_memory_dir() == tmp_path / "configured-memory" + assert handler._resolve_chroma_path() == tmp_path / "configured-chroma" + + +def test_websocket_defers_memory_export_to_core_dispatch(monkeypatch): + config = types.ModuleType("config") + config.config_get_by_key = lambda key, default=None: default + logger_mod = types.ModuleType("src.logger") + logger_mod.get_logger = lambda name: __import__("logging").getLogger(name) + channels = types.ModuleType("channels") + channels.CommChannel = object + channels.registerCommChannel = lambda *args: None + monkeypatch.setitem(sys.modules, "config", config) + monkeypatch.setitem(sys.modules, "src.logger", logger_mod) + monkeypatch.setitem(sys.modules, "channels", channels) + + spec = importlib.util.spec_from_file_location( + "wschat_under_test", REPO_ROOT / "channels" / "wschat.py" + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + received = [] + replies = [] + module._enqueue_user_message = lambda *args: received.append(args) + module.send_message = replies.append + + module._handle_frame(json.dumps({ + "type": "user_message", "seq": 1, "text": "/memory-export both" + })) + assert received == [(1, "/memory-export both")] + assert replies == [] + + +def test_commchannel_receive_dispatches_control_commands(monkeypatch): + authenticated_user_id = "telegram-user-123" + auth = types.ModuleType("auth") + auth.is_auth_enabled = lambda: True + auth.get_channel_authenticated_user_id = lambda channel: ( + authenticated_user_id if channel == "TELEGRAM" else None + ) + monkeypatch.setitem(sys.modules, "auth", auth) + + principals: list[str] = [] + control = types.ModuleType("src.memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda text, principal: ( + principals.append(principal) or "Memory export complete" + ) + monkeypatch.setitem(sys.modules, "src.memory_export", control) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + + replies: list[str] = [] + channels._commchannel = types.SimpleNamespace( + receive=lambda: "alice: /memory-export both | alice: hello", + send=replies.append, + ) + channels._commchannel_id = "telegram" + + assert channels.commChannelReceive() == "alice: hello" + assert principals == [authenticated_user_id] + assert replies == ["Memory export complete"] + + +def test_commchannel_receive_denies_export_without_authenticated_user(monkeypatch): + auth = types.ModuleType("auth") + auth.is_auth_enabled = lambda: False + auth.get_channel_authenticated_user_id = lambda *_: pytest.fail( + "disabled authentication must not resolve a user ID" + ) + monkeypatch.setitem(sys.modules, "auth", auth) + + control = types.ModuleType("src.memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda text, principal: ( + "Memory export denied: an authenticated user is required." + if principal is None + else pytest.fail("an unauthenticated command received a principal") + ) + monkeypatch.setitem(sys.modules, "src.memory_export", control) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + + replies: list[str] = [] + channels._commchannel = types.SimpleNamespace( + receive=lambda: "alice: /memory-export both", + send=replies.append, + ) + channels._commchannel_id = "telegram" + + assert channels.commChannelReceive() == "" + assert replies == ["Memory export denied: an authenticated user is required."] + + +def test_commchannel_receive_dispatches_websocket_export(monkeypatch): + websocket_token = "private-websocket-token" + config = types.ModuleType("config") + config.config_get_by_key = lambda key, default=None: ( + websocket_token if key == "WS_TOKEN" else default + ) + monkeypatch.setitem(sys.modules, "config", config) + + commands: list[str] = [] + principals: list[str] = [] + control = types.ModuleType("src.memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda text, principal: ( + commands.append(text) + or principals.append(principal) + or "Memory export complete" + ) + monkeypatch.setitem(sys.modules, "src.memory_export", control) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + + replies: list[str] = [] + channels._commchannel = types.SimpleNamespace( + receive=lambda: "/memory-export both", + send=replies.append, + ) + channels._commchannel_id = "websocket" + + assert channels.commChannelReceive() == "" + assert commands == ["/memory-export both"] + assert principals == [ + f"websocket:{hashlib.sha256(websocket_token.encode('utf-8')).hexdigest()}" + ] + assert websocket_token not in principals[0] + assert replies == ["Memory export complete"] + + +def test_websocket_export_requires_bearer_token(monkeypatch): + config = types.ModuleType("config") + config.config_get_by_key = lambda key, default=None: default + monkeypatch.setitem(sys.modules, "config", config) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + channels._commchannel_id = "websocket" + + assert channels._authenticated_export_principal() is None + + +def test_commchannel_receive_does_not_consume_command_mentions(monkeypatch): + control = types.ModuleType("src.memory_export") + control.is_export_command = lambda text: text == "/memory-export both" + control.handle_export_command = lambda *_: pytest.fail( + "a command mentioned in normal text must not execute" + ) + monkeypatch.setitem(sys.modules, "src.memory_export", control) + + monkeypatch.delitem(sys.modules, "channels", raising=False) + channels = importlib.import_module("channels") + + message = "alice: please use /memory-export both" + channels._commchannel = types.SimpleNamespace( + receive=lambda: message, + send=lambda *_: pytest.fail("normal messages must not generate replies"), + ) + channels._commchannel_id = "telegram" + + assert channels.commChannelReceive() == message diff --git a/tests/test_omegaclaw_launcher.py b/tests/test_omegaclaw_launcher.py new file mode 100644 index 00000000..655a3b18 --- /dev/null +++ b/tests/test_omegaclaw_launcher.py @@ -0,0 +1,83 @@ +import os +import subprocess +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +LAUNCHER = REPO_ROOT / "scripts" / "omega" + + +def _run_launcher(tmp_path: Path, *component_options: str) -> subprocess.CompletedProcess: + archive = tmp_path / "memory.tar.gz" + archive.touch() + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text( + "#!/bin/sh\n" + "printf 'docker'\n" + "printf ' <%s>' \"$@\"\n" + "printf '\\n'\n", + encoding="utf-8", + ) + docker.chmod(0o755) + + environment = os.environ.copy() + environment["ASI_API_KEY"] = "test-token" + environment["PATH"] = f"{bin_dir}{os.pathsep}{environment['PATH']}" + + return subprocess.run( + [ + str(LAUNCHER), + "start", + "--memory-transfer-dir", + str(tmp_path), + "--memory-import", + archive.name, + *component_options, + ], + cwd=REPO_ROOT, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + +@pytest.mark.parametrize( + ("option", "included_environment", "excluded_environment"), + [ + ("--only-history", "MEMORY_IMPORT_NO_VECTOR=1", "MEMORY_IMPORT_NO_HISTORY=1"), + ("--only-vector", "MEMORY_IMPORT_NO_HISTORY=1", "MEMORY_IMPORT_NO_VECTOR=1"), + ], +) +def test_only_component_options_select_one_import_component( + tmp_path, + option, + included_environment, + excluded_environment, +): + result = _run_launcher(tmp_path, option) + + assert result.returncode == 0, result.stderr + assert included_environment in result.stdout + assert excluded_environment not in result.stdout + + +def test_only_component_options_are_mutually_exclusive(tmp_path): + result = _run_launcher(tmp_path, "--only-history", "--only-vector") + + assert result.returncode != 0 + assert "--only-history and --only-vector cannot be combined" in result.stderr + + +@pytest.mark.parametrize("removed_option", ["--no-history", "--no-vector"]) +def test_removed_component_options_are_rejected(tmp_path, removed_option): + result = _run_launcher(tmp_path, removed_option) + + assert result.returncode != 0 + assert "Usage:" in result.stdout + assert "docker <" not in result.stdout